diff --git a/lib/python3.1/multiprocessing/__init__.py b/lib/python3.1/multiprocessing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8336f381decaacf58640640a6f05a45ba3a2c3e4 --- /dev/null +++ b/lib/python3.1/multiprocessing/__init__.py @@ -0,0 +1,37 @@ +# +# Package analogous to 'threading.py' but using processes +# +# multiprocessing/__init__.py +# +# This package is intended to duplicate the functionality (and much of +# the API) of threading.py but uses processes instead of threads. A +# subpackage 'multiprocessing.dummy' has the same API but is a simple +# wrapper for 'threading'. +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +import sys +from . import context + +# +# Copy stuff from default context +# + +__all__ = [x for x in dir(context._default_context) if not x.startswith('_')] +globals().update((name, getattr(context._default_context, name)) for name in __all__) + +# +# XXX These should not really be documented or public. +# + +SUBDEBUG = 5 +SUBWARNING = 25 + +# +# Alias for main module -- will be reset by bootstrapping child processes +# + +if '__main__' in sys.modules: + sys.modules['__mp_main__'] = sys.modules['__main__'] diff --git a/lib/python3.1/multiprocessing/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9ca0254497e8d5de2997a8f0488c11681952b0ec Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/__init__.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/connection.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/connection.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..97205d150d7000cee69dd68878ae1b9b9d46b8a8 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/connection.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/context.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/context.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d1b5745ac5ea39bf7fcab67356850e7c16ee59a Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/context.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/forkserver.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/forkserver.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7ac55c74b4c6cd86ec4de2b2e92a1be9be1fba2 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/forkserver.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/heap.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/heap.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1e4839b79ecccb841065427d661c1450171ab2e4 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/heap.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/managers.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/managers.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1317b28ed23ace850b6f9fbda7b1f887bed4d3e4 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/managers.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/pool.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/pool.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5b0596411058d613218f231cae3c5201096bc41e Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/pool.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/popen_fork.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/popen_fork.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c5e6cf0556b62a3bf0a506b10e0ca265897e2672 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/popen_fork.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/popen_forkserver.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/popen_forkserver.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02ad8e17a32c0d0a2aebf9fc82cecc36e59bd4c1 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/popen_forkserver.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/popen_spawn_posix.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/popen_spawn_posix.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..76914bd62e6874222005d6a0f9ab709c174ce64e Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/popen_spawn_posix.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/popen_spawn_win32.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/popen_spawn_win32.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..094f6a5aa862cf834f71bdd2bd1b60b05e42f999 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/popen_spawn_win32.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/process.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/process.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f35c5e583f16519f109848694642e4a62695ef1 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/process.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/queues.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/queues.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..335ae4e5c24baec7ed6c7a9a7b5e4a0ec3f2e9d9 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/queues.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/reduction.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/reduction.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..99fa1e973b24d1a4d92f555eb36569396ef114fb Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/reduction.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/resource_sharer.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/resource_sharer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3809fd8f9d26e1276ed0452d8cd96bcd7a6ebde9 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/resource_sharer.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/resource_tracker.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/resource_tracker.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e39960e2cb18b62710da54d89adab14f0ca55943 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/resource_tracker.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/shared_memory.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/shared_memory.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..051e4913ca6ace3f4ad660d04756d5dfeecd81f9 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/shared_memory.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/sharedctypes.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/sharedctypes.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aafc6e6fba2ad70a0651c61f74352cd680c0afb8 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/sharedctypes.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/spawn.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/spawn.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..14d57054e437ffd8391bb6e365b915b24d86a941 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/spawn.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/synchronize.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/synchronize.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f5a23c293d725be660668b0f5d6083deb8728401 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/synchronize.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/__pycache__/util.cpython-310.pyc b/lib/python3.1/multiprocessing/__pycache__/util.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..befa71fcc1d392f2fe87e2ae6d531d1391c905d9 Binary files /dev/null and b/lib/python3.1/multiprocessing/__pycache__/util.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/connection.py b/lib/python3.1/multiprocessing/connection.py new file mode 100644 index 0000000000000000000000000000000000000000..8e2facf92a94aa363a653a0032da4ea063f0162c --- /dev/null +++ b/lib/python3.1/multiprocessing/connection.py @@ -0,0 +1,973 @@ +# +# A higher level module for using sockets (or Windows named pipes) +# +# multiprocessing/connection.py +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +__all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ] + +import io +import os +import sys +import socket +import struct +import time +import tempfile +import itertools + +import _multiprocessing + +from . import util + +from . import AuthenticationError, BufferTooShort +from .context import reduction +_ForkingPickler = reduction.ForkingPickler + +try: + import _winapi + from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE +except ImportError: + if sys.platform == 'win32': + raise + _winapi = None + +# +# +# + +BUFSIZE = 8192 +# A very generous timeout when it comes to local connections... +CONNECTION_TIMEOUT = 20. + +_mmap_counter = itertools.count() + +default_family = 'AF_INET' +families = ['AF_INET'] + +if hasattr(socket, 'AF_UNIX'): + default_family = 'AF_UNIX' + families += ['AF_UNIX'] + +if sys.platform == 'win32': + default_family = 'AF_PIPE' + families += ['AF_PIPE'] + + +def _init_timeout(timeout=CONNECTION_TIMEOUT): + return time.monotonic() + timeout + +def _check_timeout(t): + return time.monotonic() > t + +# +# +# + +def arbitrary_address(family): + ''' + Return an arbitrary free address for the given family + ''' + if family == 'AF_INET': + return ('localhost', 0) + elif family == 'AF_UNIX': + return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir()) + elif family == 'AF_PIPE': + return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' % + (os.getpid(), next(_mmap_counter)), dir="") + else: + raise ValueError('unrecognized family') + +def _validate_family(family): + ''' + Checks if the family is valid for the current environment. + ''' + if sys.platform != 'win32' and family == 'AF_PIPE': + raise ValueError('Family %s is not recognized.' % family) + + if sys.platform == 'win32' and family == 'AF_UNIX': + # double check + if not hasattr(socket, family): + raise ValueError('Family %s is not recognized.' % family) + +def address_type(address): + ''' + Return the types of the address + + This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE' + ''' + if type(address) == tuple: + return 'AF_INET' + elif type(address) is str and address.startswith('\\\\'): + return 'AF_PIPE' + elif type(address) is str or util.is_abstract_socket_namespace(address): + return 'AF_UNIX' + else: + raise ValueError('address type of %r unrecognized' % address) + +# +# Connection classes +# + +class _ConnectionBase: + _handle = None + + def __init__(self, handle, readable=True, writable=True): + handle = handle.__index__() + if handle < 0: + raise ValueError("invalid handle") + if not readable and not writable: + raise ValueError( + "at least one of `readable` and `writable` must be True") + self._handle = handle + self._readable = readable + self._writable = writable + + # XXX should we use util.Finalize instead of a __del__? + + def __del__(self): + if self._handle is not None: + self._close() + + def _check_closed(self): + if self._handle is None: + raise OSError("handle is closed") + + def _check_readable(self): + if not self._readable: + raise OSError("connection is write-only") + + def _check_writable(self): + if not self._writable: + raise OSError("connection is read-only") + + def _bad_message_length(self): + if self._writable: + self._readable = False + else: + self.close() + raise OSError("bad message length") + + @property + def closed(self): + """True if the connection is closed""" + return self._handle is None + + @property + def readable(self): + """True if the connection is readable""" + return self._readable + + @property + def writable(self): + """True if the connection is writable""" + return self._writable + + def fileno(self): + """File descriptor or handle of the connection""" + self._check_closed() + return self._handle + + def close(self): + """Close the connection""" + if self._handle is not None: + try: + self._close() + finally: + self._handle = None + + def send_bytes(self, buf, offset=0, size=None): + """Send the bytes data from a bytes-like object""" + self._check_closed() + self._check_writable() + m = memoryview(buf) + # HACK for byte-indexing of non-bytewise buffers (e.g. array.array) + if m.itemsize > 1: + m = memoryview(bytes(m)) + n = len(m) + if offset < 0: + raise ValueError("offset is negative") + if n < offset: + raise ValueError("buffer length < offset") + if size is None: + size = n - offset + elif size < 0: + raise ValueError("size is negative") + elif offset + size > n: + raise ValueError("buffer length < offset + size") + self._send_bytes(m[offset:offset + size]) + + def send(self, obj): + """Send a (picklable) object""" + self._check_closed() + self._check_writable() + self._send_bytes(_ForkingPickler.dumps(obj)) + + def recv_bytes(self, maxlength=None): + """ + Receive bytes data as a bytes object. + """ + self._check_closed() + self._check_readable() + if maxlength is not None and maxlength < 0: + raise ValueError("negative maxlength") + buf = self._recv_bytes(maxlength) + if buf is None: + self._bad_message_length() + return buf.getvalue() + + def recv_bytes_into(self, buf, offset=0): + """ + Receive bytes data into a writeable bytes-like object. + Return the number of bytes read. + """ + self._check_closed() + self._check_readable() + with memoryview(buf) as m: + # Get bytesize of arbitrary buffer + itemsize = m.itemsize + bytesize = itemsize * len(m) + if offset < 0: + raise ValueError("negative offset") + elif offset > bytesize: + raise ValueError("offset too large") + result = self._recv_bytes() + size = result.tell() + if bytesize < offset + size: + raise BufferTooShort(result.getvalue()) + # Message can fit in dest + result.seek(0) + result.readinto(m[offset // itemsize : + (offset + size) // itemsize]) + return size + + def recv(self): + """Receive a (picklable) object""" + self._check_closed() + self._check_readable() + buf = self._recv_bytes() + return _ForkingPickler.loads(buf.getbuffer()) + + def poll(self, timeout=0.0): + """Whether there is any input available to be read""" + self._check_closed() + self._check_readable() + return self._poll(timeout) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + self.close() + + +if _winapi: + + class PipeConnection(_ConnectionBase): + """ + Connection class based on a Windows named pipe. + Overlapped I/O is used, so the handles must have been created + with FILE_FLAG_OVERLAPPED. + """ + _got_empty_message = False + + def _close(self, _CloseHandle=_winapi.CloseHandle): + _CloseHandle(self._handle) + + def _send_bytes(self, buf): + ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True) + try: + if err == _winapi.ERROR_IO_PENDING: + waitres = _winapi.WaitForMultipleObjects( + [ov.event], False, INFINITE) + assert waitres == WAIT_OBJECT_0 + except: + ov.cancel() + raise + finally: + nwritten, err = ov.GetOverlappedResult(True) + assert err == 0 + assert nwritten == len(buf) + + def _recv_bytes(self, maxsize=None): + if self._got_empty_message: + self._got_empty_message = False + return io.BytesIO() + else: + bsize = 128 if maxsize is None else min(maxsize, 128) + try: + ov, err = _winapi.ReadFile(self._handle, bsize, + overlapped=True) + try: + if err == _winapi.ERROR_IO_PENDING: + waitres = _winapi.WaitForMultipleObjects( + [ov.event], False, INFINITE) + assert waitres == WAIT_OBJECT_0 + except: + ov.cancel() + raise + finally: + nread, err = ov.GetOverlappedResult(True) + if err == 0: + f = io.BytesIO() + f.write(ov.getbuffer()) + return f + elif err == _winapi.ERROR_MORE_DATA: + return self._get_more_data(ov, maxsize) + except OSError as e: + if e.winerror == _winapi.ERROR_BROKEN_PIPE: + raise EOFError + else: + raise + raise RuntimeError("shouldn't get here; expected KeyboardInterrupt") + + def _poll(self, timeout): + if (self._got_empty_message or + _winapi.PeekNamedPipe(self._handle)[0] != 0): + return True + return bool(wait([self], timeout)) + + def _get_more_data(self, ov, maxsize): + buf = ov.getbuffer() + f = io.BytesIO() + f.write(buf) + left = _winapi.PeekNamedPipe(self._handle)[1] + assert left > 0 + if maxsize is not None and len(buf) + left > maxsize: + self._bad_message_length() + ov, err = _winapi.ReadFile(self._handle, left, overlapped=True) + rbytes, err = ov.GetOverlappedResult(True) + assert err == 0 + assert rbytes == left + f.write(ov.getbuffer()) + return f + + +class Connection(_ConnectionBase): + """ + Connection class based on an arbitrary file descriptor (Unix only), or + a socket handle (Windows). + """ + + if _winapi: + def _close(self, _close=_multiprocessing.closesocket): + _close(self._handle) + _write = _multiprocessing.send + _read = _multiprocessing.recv + else: + def _close(self, _close=os.close): + _close(self._handle) + _write = os.write + _read = os.read + + def _send(self, buf, write=_write): + remaining = len(buf) + while True: + n = write(self._handle, buf) + remaining -= n + if remaining == 0: + break + buf = buf[n:] + + def _recv(self, size, read=_read): + buf = io.BytesIO() + handle = self._handle + remaining = size + while remaining > 0: + chunk = read(handle, remaining) + n = len(chunk) + if n == 0: + if remaining == size: + raise EOFError + else: + raise OSError("got end of file during message") + buf.write(chunk) + remaining -= n + return buf + + def _send_bytes(self, buf): + n = len(buf) + if n > 0x7fffffff: + pre_header = struct.pack("!i", -1) + header = struct.pack("!Q", n) + self._send(pre_header) + self._send(header) + self._send(buf) + else: + # For wire compatibility with 3.7 and lower + header = struct.pack("!i", n) + if n > 16384: + # The payload is large so Nagle's algorithm won't be triggered + # and we'd better avoid the cost of concatenation. + self._send(header) + self._send(buf) + else: + # Issue #20540: concatenate before sending, to avoid delays due + # to Nagle's algorithm on a TCP socket. + # Also note we want to avoid sending a 0-length buffer separately, + # to avoid "broken pipe" errors if the other end closed the pipe. + self._send(header + buf) + + def _recv_bytes(self, maxsize=None): + buf = self._recv(4) + size, = struct.unpack("!i", buf.getvalue()) + if size == -1: + buf = self._recv(8) + size, = struct.unpack("!Q", buf.getvalue()) + if maxsize is not None and size > maxsize: + return None + return self._recv(size) + + def _poll(self, timeout): + r = wait([self], timeout) + return bool(r) + + +# +# Public functions +# + +class Listener(object): + ''' + Returns a listener object. + + This is a wrapper for a bound socket which is 'listening' for + connections, or for a Windows named pipe. + ''' + def __init__(self, address=None, family=None, backlog=1, authkey=None): + family = family or (address and address_type(address)) \ + or default_family + address = address or arbitrary_address(family) + + _validate_family(family) + if family == 'AF_PIPE': + self._listener = PipeListener(address, backlog) + else: + self._listener = SocketListener(address, family, backlog) + + if authkey is not None and not isinstance(authkey, bytes): + raise TypeError('authkey should be a byte string') + + self._authkey = authkey + + def accept(self): + ''' + Accept a connection on the bound socket or named pipe of `self`. + + Returns a `Connection` object. + ''' + if self._listener is None: + raise OSError('listener is closed') + c = self._listener.accept() + if self._authkey: + deliver_challenge(c, self._authkey) + answer_challenge(c, self._authkey) + return c + + def close(self): + ''' + Close the bound socket or named pipe of `self`. + ''' + listener = self._listener + if listener is not None: + self._listener = None + listener.close() + + @property + def address(self): + return self._listener._address + + @property + def last_accepted(self): + return self._listener._last_accepted + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + self.close() + + +def Client(address, family=None, authkey=None): + ''' + Returns a connection to the address of a `Listener` + ''' + family = family or address_type(address) + _validate_family(family) + if family == 'AF_PIPE': + c = PipeClient(address) + else: + c = SocketClient(address) + + if authkey is not None and not isinstance(authkey, bytes): + raise TypeError('authkey should be a byte string') + + if authkey is not None: + answer_challenge(c, authkey) + deliver_challenge(c, authkey) + + return c + + +if sys.platform != 'win32': + + def Pipe(duplex=True): + ''' + Returns pair of connection objects at either end of a pipe + ''' + if duplex: + s1, s2 = socket.socketpair() + s1.setblocking(True) + s2.setblocking(True) + c1 = Connection(s1.detach()) + c2 = Connection(s2.detach()) + else: + fd1, fd2 = os.pipe() + c1 = Connection(fd1, writable=False) + c2 = Connection(fd2, readable=False) + + return c1, c2 + +else: + + def Pipe(duplex=True): + ''' + Returns pair of connection objects at either end of a pipe + ''' + address = arbitrary_address('AF_PIPE') + if duplex: + openmode = _winapi.PIPE_ACCESS_DUPLEX + access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE + obsize, ibsize = BUFSIZE, BUFSIZE + else: + openmode = _winapi.PIPE_ACCESS_INBOUND + access = _winapi.GENERIC_WRITE + obsize, ibsize = 0, BUFSIZE + + h1 = _winapi.CreateNamedPipe( + address, openmode | _winapi.FILE_FLAG_OVERLAPPED | + _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE, + _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | + _winapi.PIPE_WAIT, + 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, + # default security descriptor: the handle cannot be inherited + _winapi.NULL + ) + h2 = _winapi.CreateFile( + address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, + _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL + ) + _winapi.SetNamedPipeHandleState( + h2, _winapi.PIPE_READMODE_MESSAGE, None, None + ) + + overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True) + _, err = overlapped.GetOverlappedResult(True) + assert err == 0 + + c1 = PipeConnection(h1, writable=duplex) + c2 = PipeConnection(h2, readable=duplex) + + return c1, c2 + +# +# Definitions for connections based on sockets +# + +class SocketListener(object): + ''' + Representation of a socket which is bound to an address and listening + ''' + def __init__(self, address, family, backlog=1): + self._socket = socket.socket(getattr(socket, family)) + try: + # SO_REUSEADDR has different semantics on Windows (issue #2550). + if os.name == 'posix': + self._socket.setsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR, 1) + self._socket.setblocking(True) + self._socket.bind(address) + self._socket.listen(backlog) + self._address = self._socket.getsockname() + except OSError: + self._socket.close() + raise + self._family = family + self._last_accepted = None + + if family == 'AF_UNIX' and not util.is_abstract_socket_namespace(address): + # Linux abstract socket namespaces do not need to be explicitly unlinked + self._unlink = util.Finalize( + self, os.unlink, args=(address,), exitpriority=0 + ) + else: + self._unlink = None + + def accept(self): + s, self._last_accepted = self._socket.accept() + s.setblocking(True) + return Connection(s.detach()) + + def close(self): + try: + self._socket.close() + finally: + unlink = self._unlink + if unlink is not None: + self._unlink = None + unlink() + + +def SocketClient(address): + ''' + Return a connection object connected to the socket given by `address` + ''' + family = address_type(address) + with socket.socket( getattr(socket, family) ) as s: + s.setblocking(True) + s.connect(address) + return Connection(s.detach()) + +# +# Definitions for connections based on named pipes +# + +if sys.platform == 'win32': + + class PipeListener(object): + ''' + Representation of a named pipe + ''' + def __init__(self, address, backlog=None): + self._address = address + self._handle_queue = [self._new_handle(first=True)] + + self._last_accepted = None + util.sub_debug('listener created with address=%r', self._address) + self.close = util.Finalize( + self, PipeListener._finalize_pipe_listener, + args=(self._handle_queue, self._address), exitpriority=0 + ) + + def _new_handle(self, first=False): + flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED + if first: + flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE + return _winapi.CreateNamedPipe( + self._address, flags, + _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | + _winapi.PIPE_WAIT, + _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE, + _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL + ) + + def accept(self): + self._handle_queue.append(self._new_handle()) + handle = self._handle_queue.pop(0) + try: + ov = _winapi.ConnectNamedPipe(handle, overlapped=True) + except OSError as e: + if e.winerror != _winapi.ERROR_NO_DATA: + raise + # ERROR_NO_DATA can occur if a client has already connected, + # written data and then disconnected -- see Issue 14725. + else: + try: + res = _winapi.WaitForMultipleObjects( + [ov.event], False, INFINITE) + except: + ov.cancel() + _winapi.CloseHandle(handle) + raise + finally: + _, err = ov.GetOverlappedResult(True) + assert err == 0 + return PipeConnection(handle) + + @staticmethod + def _finalize_pipe_listener(queue, address): + util.sub_debug('closing listener with address=%r', address) + for handle in queue: + _winapi.CloseHandle(handle) + + def PipeClient(address): + ''' + Return a connection object connected to the pipe given by `address` + ''' + t = _init_timeout() + while 1: + try: + _winapi.WaitNamedPipe(address, 1000) + h = _winapi.CreateFile( + address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE, + 0, _winapi.NULL, _winapi.OPEN_EXISTING, + _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL + ) + except OSError as e: + if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT, + _winapi.ERROR_PIPE_BUSY) or _check_timeout(t): + raise + else: + break + else: + raise + + _winapi.SetNamedPipeHandleState( + h, _winapi.PIPE_READMODE_MESSAGE, None, None + ) + return PipeConnection(h) + +# +# Authentication stuff +# + +MESSAGE_LENGTH = 20 + +CHALLENGE = b'#CHALLENGE#' +WELCOME = b'#WELCOME#' +FAILURE = b'#FAILURE#' + +def deliver_challenge(connection, authkey): + import hmac + if not isinstance(authkey, bytes): + raise ValueError( + "Authkey must be bytes, not {0!s}".format(type(authkey))) + message = os.urandom(MESSAGE_LENGTH) + connection.send_bytes(CHALLENGE + message) + digest = hmac.new(authkey, message, 'md5').digest() + response = connection.recv_bytes(256) # reject large message + if response == digest: + connection.send_bytes(WELCOME) + else: + connection.send_bytes(FAILURE) + raise AuthenticationError('digest received was wrong') + +def answer_challenge(connection, authkey): + import hmac + if not isinstance(authkey, bytes): + raise ValueError( + "Authkey must be bytes, not {0!s}".format(type(authkey))) + message = connection.recv_bytes(256) # reject large message + assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message + message = message[len(CHALLENGE):] + digest = hmac.new(authkey, message, 'md5').digest() + connection.send_bytes(digest) + response = connection.recv_bytes(256) # reject large message + if response != WELCOME: + raise AuthenticationError('digest sent was rejected') + +# +# Support for using xmlrpclib for serialization +# + +class ConnectionWrapper(object): + def __init__(self, conn, dumps, loads): + self._conn = conn + self._dumps = dumps + self._loads = loads + for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'): + obj = getattr(conn, attr) + setattr(self, attr, obj) + def send(self, obj): + s = self._dumps(obj) + self._conn.send_bytes(s) + def recv(self): + s = self._conn.recv_bytes() + return self._loads(s) + +def _xml_dumps(obj): + return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8') + +def _xml_loads(s): + (obj,), method = xmlrpclib.loads(s.decode('utf-8')) + return obj + +class XmlListener(Listener): + def accept(self): + global xmlrpclib + import xmlrpc.client as xmlrpclib + obj = Listener.accept(self) + return ConnectionWrapper(obj, _xml_dumps, _xml_loads) + +def XmlClient(*args, **kwds): + global xmlrpclib + import xmlrpc.client as xmlrpclib + return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads) + +# +# Wait +# + +if sys.platform == 'win32': + + def _exhaustive_wait(handles, timeout): + # Return ALL handles which are currently signalled. (Only + # returning the first signalled might create starvation issues.) + L = list(handles) + ready = [] + while L: + res = _winapi.WaitForMultipleObjects(L, False, timeout) + if res == WAIT_TIMEOUT: + break + elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L): + res -= WAIT_OBJECT_0 + elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L): + res -= WAIT_ABANDONED_0 + else: + raise RuntimeError('Should not get here') + ready.append(L[res]) + L = L[res+1:] + timeout = 0 + return ready + + _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED} + + def wait(object_list, timeout=None): + ''' + Wait till an object in object_list is ready/readable. + + Returns list of those objects in object_list which are ready/readable. + ''' + if timeout is None: + timeout = INFINITE + elif timeout < 0: + timeout = 0 + else: + timeout = int(timeout * 1000 + 0.5) + + object_list = list(object_list) + waithandle_to_obj = {} + ov_list = [] + ready_objects = set() + ready_handles = set() + + try: + for o in object_list: + try: + fileno = getattr(o, 'fileno') + except AttributeError: + waithandle_to_obj[o.__index__()] = o + else: + # start an overlapped read of length zero + try: + ov, err = _winapi.ReadFile(fileno(), 0, True) + except OSError as e: + ov, err = None, e.winerror + if err not in _ready_errors: + raise + if err == _winapi.ERROR_IO_PENDING: + ov_list.append(ov) + waithandle_to_obj[ov.event] = o + else: + # If o.fileno() is an overlapped pipe handle and + # err == 0 then there is a zero length message + # in the pipe, but it HAS NOT been consumed... + if ov and sys.getwindowsversion()[:2] >= (6, 2): + # ... except on Windows 8 and later, where + # the message HAS been consumed. + try: + _, err = ov.GetOverlappedResult(False) + except OSError as e: + err = e.winerror + if not err and hasattr(o, '_got_empty_message'): + o._got_empty_message = True + ready_objects.add(o) + timeout = 0 + + ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout) + finally: + # request that overlapped reads stop + for ov in ov_list: + ov.cancel() + + # wait for all overlapped reads to stop + for ov in ov_list: + try: + _, err = ov.GetOverlappedResult(True) + except OSError as e: + err = e.winerror + if err not in _ready_errors: + raise + if err != _winapi.ERROR_OPERATION_ABORTED: + o = waithandle_to_obj[ov.event] + ready_objects.add(o) + if err == 0: + # If o.fileno() is an overlapped pipe handle then + # a zero length message HAS been consumed. + if hasattr(o, '_got_empty_message'): + o._got_empty_message = True + + ready_objects.update(waithandle_to_obj[h] for h in ready_handles) + return [o for o in object_list if o in ready_objects] + +else: + + import selectors + + # poll/select have the advantage of not requiring any extra file + # descriptor, contrarily to epoll/kqueue (also, they require a single + # syscall). + if hasattr(selectors, 'PollSelector'): + _WaitSelector = selectors.PollSelector + else: + _WaitSelector = selectors.SelectSelector + + def wait(object_list, timeout=None): + ''' + Wait till an object in object_list is ready/readable. + + Returns list of those objects in object_list which are ready/readable. + ''' + with _WaitSelector() as selector: + for obj in object_list: + selector.register(obj, selectors.EVENT_READ) + + if timeout is not None: + deadline = time.monotonic() + timeout + + while True: + ready = selector.select(timeout) + if ready: + return [key.fileobj for (key, events) in ready] + else: + if timeout is not None: + timeout = deadline - time.monotonic() + if timeout < 0: + return ready + +# +# Make connection and socket objects sharable if possible +# + +if sys.platform == 'win32': + def reduce_connection(conn): + handle = conn.fileno() + with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s: + from . import resource_sharer + ds = resource_sharer.DupSocket(s) + return rebuild_connection, (ds, conn.readable, conn.writable) + def rebuild_connection(ds, readable, writable): + sock = ds.detach() + return Connection(sock.detach(), readable, writable) + reduction.register(Connection, reduce_connection) + + def reduce_pipe_connection(conn): + access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) | + (_winapi.FILE_GENERIC_WRITE if conn.writable else 0)) + dh = reduction.DupHandle(conn.fileno(), access) + return rebuild_pipe_connection, (dh, conn.readable, conn.writable) + def rebuild_pipe_connection(dh, readable, writable): + handle = dh.detach() + return PipeConnection(handle, readable, writable) + reduction.register(PipeConnection, reduce_pipe_connection) + +else: + def reduce_connection(conn): + df = reduction.DupFd(conn.fileno()) + return rebuild_connection, (df, conn.readable, conn.writable) + def rebuild_connection(df, readable, writable): + fd = df.detach() + return Connection(fd, readable, writable) + reduction.register(Connection, reduce_connection) diff --git a/lib/python3.1/multiprocessing/context.py b/lib/python3.1/multiprocessing/context.py new file mode 100644 index 0000000000000000000000000000000000000000..b1960ea296fe208935b706898f3d564b9a598d6e --- /dev/null +++ b/lib/python3.1/multiprocessing/context.py @@ -0,0 +1,376 @@ +import os +import sys +import threading + +from . import process +from . import reduction + +__all__ = () + +# +# Exceptions +# + +class ProcessError(Exception): + pass + +class BufferTooShort(ProcessError): + pass + +class TimeoutError(ProcessError): + pass + +class AuthenticationError(ProcessError): + pass + +# +# Base type for contexts. Bound methods of an instance of this type are included in __all__ of __init__.py +# + +class BaseContext(object): + + ProcessError = ProcessError + BufferTooShort = BufferTooShort + TimeoutError = TimeoutError + AuthenticationError = AuthenticationError + + current_process = staticmethod(process.current_process) + parent_process = staticmethod(process.parent_process) + active_children = staticmethod(process.active_children) + + def cpu_count(self): + '''Returns the number of CPUs in the system''' + num = os.cpu_count() + if num is None: + raise NotImplementedError('cannot determine number of cpus') + else: + return num + + def Manager(self): + '''Returns a manager associated with a running server process + + The managers methods such as `Lock()`, `Condition()` and `Queue()` + can be used to create shared objects. + ''' + from .managers import SyncManager + m = SyncManager(ctx=self.get_context()) + m.start() + return m + + def Pipe(self, duplex=True): + '''Returns two connection object connected by a pipe''' + from .connection import Pipe + return Pipe(duplex) + + def Lock(self): + '''Returns a non-recursive lock object''' + from .synchronize import Lock + return Lock(ctx=self.get_context()) + + def RLock(self): + '''Returns a recursive lock object''' + from .synchronize import RLock + return RLock(ctx=self.get_context()) + + def Condition(self, lock=None): + '''Returns a condition object''' + from .synchronize import Condition + return Condition(lock, ctx=self.get_context()) + + def Semaphore(self, value=1): + '''Returns a semaphore object''' + from .synchronize import Semaphore + return Semaphore(value, ctx=self.get_context()) + + def BoundedSemaphore(self, value=1): + '''Returns a bounded semaphore object''' + from .synchronize import BoundedSemaphore + return BoundedSemaphore(value, ctx=self.get_context()) + + def Event(self): + '''Returns an event object''' + from .synchronize import Event + return Event(ctx=self.get_context()) + + def Barrier(self, parties, action=None, timeout=None): + '''Returns a barrier object''' + from .synchronize import Barrier + return Barrier(parties, action, timeout, ctx=self.get_context()) + + def Queue(self, maxsize=0): + '''Returns a queue object''' + from .queues import Queue + return Queue(maxsize, ctx=self.get_context()) + + def JoinableQueue(self, maxsize=0): + '''Returns a queue object''' + from .queues import JoinableQueue + return JoinableQueue(maxsize, ctx=self.get_context()) + + def SimpleQueue(self): + '''Returns a queue object''' + from .queues import SimpleQueue + return SimpleQueue(ctx=self.get_context()) + + def Pool(self, processes=None, initializer=None, initargs=(), + maxtasksperchild=None): + '''Returns a process pool object''' + from .pool import Pool + return Pool(processes, initializer, initargs, maxtasksperchild, + context=self.get_context()) + + def RawValue(self, typecode_or_type, *args): + '''Returns a shared object''' + from .sharedctypes import RawValue + return RawValue(typecode_or_type, *args) + + def RawArray(self, typecode_or_type, size_or_initializer): + '''Returns a shared array''' + from .sharedctypes import RawArray + return RawArray(typecode_or_type, size_or_initializer) + + def Value(self, typecode_or_type, *args, lock=True): + '''Returns a synchronized shared object''' + from .sharedctypes import Value + return Value(typecode_or_type, *args, lock=lock, + ctx=self.get_context()) + + def Array(self, typecode_or_type, size_or_initializer, *, lock=True): + '''Returns a synchronized shared array''' + from .sharedctypes import Array + return Array(typecode_or_type, size_or_initializer, lock=lock, + ctx=self.get_context()) + + def freeze_support(self): + '''Check whether this is a fake forked process in a frozen executable. + If so then run code specified by commandline and exit. + ''' + if sys.platform == 'win32' and getattr(sys, 'frozen', False): + from .spawn import freeze_support + freeze_support() + + def get_logger(self): + '''Return package logger -- if it does not already exist then + it is created. + ''' + from .util import get_logger + return get_logger() + + def log_to_stderr(self, level=None): + '''Turn on logging and add a handler which prints to stderr''' + from .util import log_to_stderr + return log_to_stderr(level) + + def allow_connection_pickling(self): + '''Install support for sending connections and sockets + between processes + ''' + # This is undocumented. In previous versions of multiprocessing + # its only effect was to make socket objects inheritable on Windows. + from . import connection + + def set_executable(self, executable): + '''Sets the path to a python.exe or pythonw.exe binary used to run + child processes instead of sys.executable when using the 'spawn' + start method. Useful for people embedding Python. + ''' + from .spawn import set_executable + set_executable(executable) + + def set_forkserver_preload(self, module_names): + '''Set list of module names to try to load in forkserver process. + This is really just a hint. + ''' + from .forkserver import set_forkserver_preload + set_forkserver_preload(module_names) + + def get_context(self, method=None): + if method is None: + return self + try: + ctx = _concrete_contexts[method] + except KeyError: + raise ValueError('cannot find context for %r' % method) from None + ctx._check_available() + return ctx + + def get_start_method(self, allow_none=False): + return self._name + + def set_start_method(self, method, force=False): + raise ValueError('cannot set start method of concrete context') + + @property + def reducer(self): + '''Controls how objects will be reduced to a form that can be + shared with other processes.''' + return globals().get('reduction') + + @reducer.setter + def reducer(self, reduction): + globals()['reduction'] = reduction + + def _check_available(self): + pass + +# +# Type of default context -- underlying context can be set at most once +# + +class Process(process.BaseProcess): + _start_method = None + @staticmethod + def _Popen(process_obj): + return _default_context.get_context().Process._Popen(process_obj) + + @staticmethod + def _after_fork(): + return _default_context.get_context().Process._after_fork() + +class DefaultContext(BaseContext): + Process = Process + + def __init__(self, context): + self._default_context = context + self._actual_context = None + + def get_context(self, method=None): + if method is None: + if self._actual_context is None: + self._actual_context = self._default_context + return self._actual_context + else: + return super().get_context(method) + + def set_start_method(self, method, force=False): + if self._actual_context is not None and not force: + raise RuntimeError('context has already been set') + if method is None and force: + self._actual_context = None + return + self._actual_context = self.get_context(method) + + def get_start_method(self, allow_none=False): + if self._actual_context is None: + if allow_none: + return None + self._actual_context = self._default_context + return self._actual_context._name + + def get_all_start_methods(self): + if sys.platform == 'win32': + return ['spawn'] + else: + methods = ['spawn', 'fork'] if sys.platform == 'darwin' else ['fork', 'spawn'] + if reduction.HAVE_SEND_HANDLE: + methods.append('forkserver') + return methods + + +# +# Context types for fixed start method +# + +if sys.platform != 'win32': + + class ForkProcess(process.BaseProcess): + _start_method = 'fork' + @staticmethod + def _Popen(process_obj): + from .popen_fork import Popen + return Popen(process_obj) + + class SpawnProcess(process.BaseProcess): + _start_method = 'spawn' + @staticmethod + def _Popen(process_obj): + from .popen_spawn_posix import Popen + return Popen(process_obj) + + @staticmethod + def _after_fork(): + # process is spawned, nothing to do + pass + + class ForkServerProcess(process.BaseProcess): + _start_method = 'forkserver' + @staticmethod + def _Popen(process_obj): + from .popen_forkserver import Popen + return Popen(process_obj) + + class ForkContext(BaseContext): + _name = 'fork' + Process = ForkProcess + + class SpawnContext(BaseContext): + _name = 'spawn' + Process = SpawnProcess + + class ForkServerContext(BaseContext): + _name = 'forkserver' + Process = ForkServerProcess + def _check_available(self): + if not reduction.HAVE_SEND_HANDLE: + raise ValueError('forkserver start method not available') + + _concrete_contexts = { + 'fork': ForkContext(), + 'spawn': SpawnContext(), + 'forkserver': ForkServerContext(), + } + if sys.platform == 'darwin': + # bpo-33725: running arbitrary code after fork() is no longer reliable + # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. + _default_context = DefaultContext(_concrete_contexts['spawn']) + else: + _default_context = DefaultContext(_concrete_contexts['fork']) + +else: + + class SpawnProcess(process.BaseProcess): + _start_method = 'spawn' + @staticmethod + def _Popen(process_obj): + from .popen_spawn_win32 import Popen + return Popen(process_obj) + + @staticmethod + def _after_fork(): + # process is spawned, nothing to do + pass + + class SpawnContext(BaseContext): + _name = 'spawn' + Process = SpawnProcess + + _concrete_contexts = { + 'spawn': SpawnContext(), + } + _default_context = DefaultContext(_concrete_contexts['spawn']) + +# +# Force the start method +# + +def _force_start_method(method): + _default_context._actual_context = _concrete_contexts[method] + +# +# Check that the current thread is spawning a child process +# + +_tls = threading.local() + +def get_spawning_popen(): + return getattr(_tls, 'spawning_popen', None) + +def set_spawning_popen(popen): + _tls.spawning_popen = popen + +def assert_spawning(obj): + if get_spawning_popen() is None: + raise RuntimeError( + '%s objects should only be shared between processes' + ' through inheritance' % type(obj).__name__ + ) diff --git a/lib/python3.1/multiprocessing/dummy/__init__.py b/lib/python3.1/multiprocessing/dummy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6a1468609e347b3a0b9281e5c9e6ec311fcb37e5 --- /dev/null +++ b/lib/python3.1/multiprocessing/dummy/__init__.py @@ -0,0 +1,126 @@ +# +# Support for the API of the multiprocessing package using threads +# +# multiprocessing/dummy/__init__.py +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +__all__ = [ + 'Process', 'current_process', 'active_children', 'freeze_support', + 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', + 'Event', 'Barrier', 'Queue', 'Manager', 'Pipe', 'Pool', 'JoinableQueue' + ] + +# +# Imports +# + +import threading +import sys +import weakref +import array + +from .connection import Pipe +from threading import Lock, RLock, Semaphore, BoundedSemaphore +from threading import Event, Condition, Barrier +from queue import Queue + +# +# +# + +class DummyProcess(threading.Thread): + + def __init__(self, group=None, target=None, name=None, args=(), kwargs={}): + threading.Thread.__init__(self, group, target, name, args, kwargs) + self._pid = None + self._children = weakref.WeakKeyDictionary() + self._start_called = False + self._parent = current_process() + + def start(self): + if self._parent is not current_process(): + raise RuntimeError( + "Parent is {0!r} but current_process is {1!r}".format( + self._parent, current_process())) + self._start_called = True + if hasattr(self._parent, '_children'): + self._parent._children[self] = None + threading.Thread.start(self) + + @property + def exitcode(self): + if self._start_called and not self.is_alive(): + return 0 + else: + return None + +# +# +# + +Process = DummyProcess +current_process = threading.current_thread +current_process()._children = weakref.WeakKeyDictionary() + +def active_children(): + children = current_process()._children + for p in list(children): + if not p.is_alive(): + children.pop(p, None) + return list(children) + +def freeze_support(): + pass + +# +# +# + +class Namespace(object): + def __init__(self, /, **kwds): + self.__dict__.update(kwds) + def __repr__(self): + items = list(self.__dict__.items()) + temp = [] + for name, value in items: + if not name.startswith('_'): + temp.append('%s=%r' % (name, value)) + temp.sort() + return '%s(%s)' % (self.__class__.__name__, ', '.join(temp)) + +dict = dict +list = list + +def Array(typecode, sequence, lock=True): + return array.array(typecode, sequence) + +class Value(object): + def __init__(self, typecode, value, lock=True): + self._typecode = typecode + self._value = value + + @property + def value(self): + return self._value + + @value.setter + def value(self, value): + self._value = value + + def __repr__(self): + return '<%s(%r, %r)>'%(type(self).__name__,self._typecode,self._value) + +def Manager(): + return sys.modules[__name__] + +def shutdown(): + pass + +def Pool(processes=None, initializer=None, initargs=()): + from ..pool import ThreadPool + return ThreadPool(processes, initializer, initargs) + +JoinableQueue = Queue diff --git a/lib/python3.1/multiprocessing/dummy/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/multiprocessing/dummy/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4ee8bd1218cf55741d946ed8b8b41d83b5851f3e Binary files /dev/null and b/lib/python3.1/multiprocessing/dummy/__pycache__/__init__.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/dummy/__pycache__/connection.cpython-310.pyc b/lib/python3.1/multiprocessing/dummy/__pycache__/connection.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1dda95d27715cf7051c257d447ffce542ff0ddf5 Binary files /dev/null and b/lib/python3.1/multiprocessing/dummy/__pycache__/connection.cpython-310.pyc differ diff --git a/lib/python3.1/multiprocessing/dummy/connection.py b/lib/python3.1/multiprocessing/dummy/connection.py new file mode 100644 index 0000000000000000000000000000000000000000..f0ce320fcf514083f3a6477e87abf40e9719285a --- /dev/null +++ b/lib/python3.1/multiprocessing/dummy/connection.py @@ -0,0 +1,75 @@ +# +# Analogue of `multiprocessing.connection` which uses queues instead of sockets +# +# multiprocessing/dummy/connection.py +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +__all__ = [ 'Client', 'Listener', 'Pipe' ] + +from queue import Queue + + +families = [None] + + +class Listener(object): + + def __init__(self, address=None, family=None, backlog=1): + self._backlog_queue = Queue(backlog) + + def accept(self): + return Connection(*self._backlog_queue.get()) + + def close(self): + self._backlog_queue = None + + @property + def address(self): + return self._backlog_queue + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + self.close() + + +def Client(address): + _in, _out = Queue(), Queue() + address.put((_out, _in)) + return Connection(_in, _out) + + +def Pipe(duplex=True): + a, b = Queue(), Queue() + return Connection(a, b), Connection(b, a) + + +class Connection(object): + + def __init__(self, _in, _out): + self._out = _out + self._in = _in + self.send = self.send_bytes = _out.put + self.recv = self.recv_bytes = _in.get + + def poll(self, timeout=0.0): + if self._in.qsize() > 0: + return True + if timeout <= 0.0: + return False + with self._in.not_empty: + self._in.not_empty.wait(timeout) + return self._in.qsize() > 0 + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + self.close() diff --git a/lib/python3.1/multiprocessing/forkserver.py b/lib/python3.1/multiprocessing/forkserver.py new file mode 100644 index 0000000000000000000000000000000000000000..22a911a7a29cdc0219cb46da48fa7e2081abcc83 --- /dev/null +++ b/lib/python3.1/multiprocessing/forkserver.py @@ -0,0 +1,348 @@ +import errno +import os +import selectors +import signal +import socket +import struct +import sys +import threading +import warnings + +from . import connection +from . import process +from .context import reduction +from . import resource_tracker +from . import spawn +from . import util + +__all__ = ['ensure_running', 'get_inherited_fds', 'connect_to_new_process', + 'set_forkserver_preload'] + +# +# +# + +MAXFDS_TO_SEND = 256 +SIGNED_STRUCT = struct.Struct('q') # large enough for pid_t + +# +# Forkserver class +# + +class ForkServer(object): + + def __init__(self): + self._forkserver_address = None + self._forkserver_alive_fd = None + self._forkserver_pid = None + self._inherited_fds = None + self._lock = threading.Lock() + self._preload_modules = ['__main__'] + + def _stop(self): + # Method used by unit tests to stop the server + with self._lock: + self._stop_unlocked() + + def _stop_unlocked(self): + if self._forkserver_pid is None: + return + + # close the "alive" file descriptor asks the server to stop + os.close(self._forkserver_alive_fd) + self._forkserver_alive_fd = None + + os.waitpid(self._forkserver_pid, 0) + self._forkserver_pid = None + + if not util.is_abstract_socket_namespace(self._forkserver_address): + os.unlink(self._forkserver_address) + self._forkserver_address = None + + def set_forkserver_preload(self, modules_names): + '''Set list of module names to try to load in forkserver process.''' + if not all(type(mod) is str for mod in self._preload_modules): + raise TypeError('module_names must be a list of strings') + self._preload_modules = modules_names + + def get_inherited_fds(self): + '''Return list of fds inherited from parent process. + + This returns None if the current process was not started by fork + server. + ''' + return self._inherited_fds + + def connect_to_new_process(self, fds): + '''Request forkserver to create a child process. + + Returns a pair of fds (status_r, data_w). The calling process can read + the child process's pid and (eventually) its returncode from status_r. + The calling process should write to data_w the pickled preparation and + process data. + ''' + self.ensure_running() + if len(fds) + 4 >= MAXFDS_TO_SEND: + raise ValueError('too many fds') + with socket.socket(socket.AF_UNIX) as client: + client.connect(self._forkserver_address) + parent_r, child_w = os.pipe() + child_r, parent_w = os.pipe() + allfds = [child_r, child_w, self._forkserver_alive_fd, + resource_tracker.getfd()] + allfds += fds + try: + reduction.sendfds(client, allfds) + return parent_r, parent_w + except: + os.close(parent_r) + os.close(parent_w) + raise + finally: + os.close(child_r) + os.close(child_w) + + def ensure_running(self): + '''Make sure that a fork server is running. + + This can be called from any process. Note that usually a child + process will just reuse the forkserver started by its parent, so + ensure_running() will do nothing. + ''' + with self._lock: + resource_tracker.ensure_running() + if self._forkserver_pid is not None: + # forkserver was launched before, is it still running? + pid, status = os.waitpid(self._forkserver_pid, os.WNOHANG) + if not pid: + # still alive + return + # dead, launch it again + os.close(self._forkserver_alive_fd) + self._forkserver_address = None + self._forkserver_alive_fd = None + self._forkserver_pid = None + + cmd = ('from multiprocessing.forkserver import main; ' + + 'main(%d, %d, %r, **%r)') + + if self._preload_modules: + desired_keys = {'main_path', 'sys_path'} + data = spawn.get_preparation_data('ignore') + data = {x: y for x, y in data.items() if x in desired_keys} + else: + data = {} + + with socket.socket(socket.AF_UNIX) as listener: + address = connection.arbitrary_address('AF_UNIX') + listener.bind(address) + if not util.is_abstract_socket_namespace(address): + os.chmod(address, 0o600) + listener.listen() + + # all client processes own the write end of the "alive" pipe; + # when they all terminate the read end becomes ready. + alive_r, alive_w = os.pipe() + try: + fds_to_pass = [listener.fileno(), alive_r] + cmd %= (listener.fileno(), alive_r, self._preload_modules, + data) + exe = spawn.get_executable() + args = [exe] + util._args_from_interpreter_flags() + args += ['-c', cmd] + pid = util.spawnv_passfds(exe, args, fds_to_pass) + except: + os.close(alive_w) + raise + finally: + os.close(alive_r) + self._forkserver_address = address + self._forkserver_alive_fd = alive_w + self._forkserver_pid = pid + +# +# +# + +def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): + '''Run forkserver.''' + if preload: + if '__main__' in preload and main_path is not None: + process.current_process()._inheriting = True + try: + spawn.import_main_path(main_path) + finally: + del process.current_process()._inheriting + for modname in preload: + try: + __import__(modname) + except ImportError: + pass + + util._close_stdin() + + sig_r, sig_w = os.pipe() + os.set_blocking(sig_r, False) + os.set_blocking(sig_w, False) + + def sigchld_handler(*_unused): + # Dummy signal handler, doesn't do anything + pass + + handlers = { + # unblocking SIGCHLD allows the wakeup fd to notify our event loop + signal.SIGCHLD: sigchld_handler, + # protect the process from ^C + signal.SIGINT: signal.SIG_IGN, + } + old_handlers = {sig: signal.signal(sig, val) + for (sig, val) in handlers.items()} + + # calling os.write() in the Python signal handler is racy + signal.set_wakeup_fd(sig_w) + + # map child pids to client fds + pid_to_fd = {} + + with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \ + selectors.DefaultSelector() as selector: + _forkserver._forkserver_address = listener.getsockname() + + selector.register(listener, selectors.EVENT_READ) + selector.register(alive_r, selectors.EVENT_READ) + selector.register(sig_r, selectors.EVENT_READ) + + while True: + try: + while True: + rfds = [key.fileobj for (key, events) in selector.select()] + if rfds: + break + + if alive_r in rfds: + # EOF because no more client processes left + assert os.read(alive_r, 1) == b'', "Not at EOF?" + raise SystemExit + + if sig_r in rfds: + # Got SIGCHLD + os.read(sig_r, 65536) # exhaust + while True: + # Scan for child processes + try: + pid, sts = os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + break + if pid == 0: + break + child_w = pid_to_fd.pop(pid, None) + if child_w is not None: + returncode = os.waitstatus_to_exitcode(sts) + + # Send exit code to client process + try: + write_signed(child_w, returncode) + except BrokenPipeError: + # client vanished + pass + os.close(child_w) + else: + # This shouldn't happen really + warnings.warn('forkserver: waitpid returned ' + 'unexpected pid %d' % pid) + + if listener in rfds: + # Incoming fork request + with listener.accept()[0] as s: + # Receive fds from client + fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1) + if len(fds) > MAXFDS_TO_SEND: + raise RuntimeError( + "Too many ({0:n}) fds to send".format( + len(fds))) + child_r, child_w, *fds = fds + s.close() + pid = os.fork() + if pid == 0: + # Child + code = 1 + try: + listener.close() + selector.close() + unused_fds = [alive_r, child_w, sig_r, sig_w] + unused_fds.extend(pid_to_fd.values()) + code = _serve_one(child_r, fds, + unused_fds, + old_handlers) + except Exception: + sys.excepthook(*sys.exc_info()) + sys.stderr.flush() + finally: + os._exit(code) + else: + # Send pid to client process + try: + write_signed(child_w, pid) + except BrokenPipeError: + # client vanished + pass + pid_to_fd[pid] = child_w + os.close(child_r) + for fd in fds: + os.close(fd) + + except OSError as e: + if e.errno != errno.ECONNABORTED: + raise + + +def _serve_one(child_r, fds, unused_fds, handlers): + # close unnecessary stuff and reset signal handlers + signal.set_wakeup_fd(-1) + for sig, val in handlers.items(): + signal.signal(sig, val) + for fd in unused_fds: + os.close(fd) + + (_forkserver._forkserver_alive_fd, + resource_tracker._resource_tracker._fd, + *_forkserver._inherited_fds) = fds + + # Run process object received over pipe + parent_sentinel = os.dup(child_r) + code = spawn._main(child_r, parent_sentinel) + + return code + + +# +# Read and write signed numbers +# + +def read_signed(fd): + data = b'' + length = SIGNED_STRUCT.size + while len(data) < length: + s = os.read(fd, length - len(data)) + if not s: + raise EOFError('unexpected EOF') + data += s + return SIGNED_STRUCT.unpack(data)[0] + +def write_signed(fd, n): + msg = SIGNED_STRUCT.pack(n) + while msg: + nbytes = os.write(fd, msg) + if nbytes == 0: + raise RuntimeError('should not get here') + msg = msg[nbytes:] + +# +# +# + +_forkserver = ForkServer() +ensure_running = _forkserver.ensure_running +get_inherited_fds = _forkserver.get_inherited_fds +connect_to_new_process = _forkserver.connect_to_new_process +set_forkserver_preload = _forkserver.set_forkserver_preload diff --git a/lib/python3.1/multiprocessing/heap.py b/lib/python3.1/multiprocessing/heap.py new file mode 100644 index 0000000000000000000000000000000000000000..6217dfe12689b379f2dad6f1e4bc3bbf6af8f60a --- /dev/null +++ b/lib/python3.1/multiprocessing/heap.py @@ -0,0 +1,337 @@ +# +# Module which supports allocation of memory from an mmap +# +# multiprocessing/heap.py +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +import bisect +from collections import defaultdict +import mmap +import os +import sys +import tempfile +import threading + +from .context import reduction, assert_spawning +from . import util + +__all__ = ['BufferWrapper'] + +# +# Inheritable class which wraps an mmap, and from which blocks can be allocated +# + +if sys.platform == 'win32': + + import _winapi + + class Arena(object): + """ + A shared memory area backed by anonymous memory (Windows). + """ + + _rand = tempfile._RandomNameSequence() + + def __init__(self, size): + self.size = size + for i in range(100): + name = 'pym-%d-%s' % (os.getpid(), next(self._rand)) + buf = mmap.mmap(-1, size, tagname=name) + if _winapi.GetLastError() == 0: + break + # We have reopened a preexisting mmap. + buf.close() + else: + raise FileExistsError('Cannot find name for new mmap') + self.name = name + self.buffer = buf + self._state = (self.size, self.name) + + def __getstate__(self): + assert_spawning(self) + return self._state + + def __setstate__(self, state): + self.size, self.name = self._state = state + # Reopen existing mmap + self.buffer = mmap.mmap(-1, self.size, tagname=self.name) + # XXX Temporarily preventing buildbot failures while determining + # XXX the correct long-term fix. See issue 23060 + #assert _winapi.GetLastError() == _winapi.ERROR_ALREADY_EXISTS + +else: + + class Arena(object): + """ + A shared memory area backed by a temporary file (POSIX). + """ + + if sys.platform == 'linux': + _dir_candidates = ['/dev/shm'] + else: + _dir_candidates = [] + + def __init__(self, size, fd=-1): + self.size = size + self.fd = fd + if fd == -1: + # Arena is created anew (if fd != -1, it means we're coming + # from rebuild_arena() below) + self.fd, name = tempfile.mkstemp( + prefix='pym-%d-'%os.getpid(), + dir=self._choose_dir(size)) + os.unlink(name) + util.Finalize(self, os.close, (self.fd,)) + os.ftruncate(self.fd, size) + self.buffer = mmap.mmap(self.fd, self.size) + + def _choose_dir(self, size): + # Choose a non-storage backed directory if possible, + # to improve performance + for d in self._dir_candidates: + st = os.statvfs(d) + if st.f_bavail * st.f_frsize >= size: # enough free space? + return d + return util.get_temp_dir() + + def reduce_arena(a): + if a.fd == -1: + raise ValueError('Arena is unpicklable because ' + 'forking was enabled when it was created') + return rebuild_arena, (a.size, reduction.DupFd(a.fd)) + + def rebuild_arena(size, dupfd): + return Arena(size, dupfd.detach()) + + reduction.register(Arena, reduce_arena) + +# +# Class allowing allocation of chunks of memory from arenas +# + +class Heap(object): + + # Minimum malloc() alignment + _alignment = 8 + + _DISCARD_FREE_SPACE_LARGER_THAN = 4 * 1024 ** 2 # 4 MB + _DOUBLE_ARENA_SIZE_UNTIL = 4 * 1024 ** 2 + + def __init__(self, size=mmap.PAGESIZE): + self._lastpid = os.getpid() + self._lock = threading.Lock() + # Current arena allocation size + self._size = size + # A sorted list of available block sizes in arenas + self._lengths = [] + + # Free block management: + # - map each block size to a list of `(Arena, start, stop)` blocks + self._len_to_seq = {} + # - map `(Arena, start)` tuple to the `(Arena, start, stop)` block + # starting at that offset + self._start_to_block = {} + # - map `(Arena, stop)` tuple to the `(Arena, start, stop)` block + # ending at that offset + self._stop_to_block = {} + + # Map arenas to their `(Arena, start, stop)` blocks in use + self._allocated_blocks = defaultdict(set) + self._arenas = [] + + # List of pending blocks to free - see comment in free() below + self._pending_free_blocks = [] + + # Statistics + self._n_mallocs = 0 + self._n_frees = 0 + + @staticmethod + def _roundup(n, alignment): + # alignment must be a power of 2 + mask = alignment - 1 + return (n + mask) & ~mask + + def _new_arena(self, size): + # Create a new arena with at least the given *size* + length = self._roundup(max(self._size, size), mmap.PAGESIZE) + # We carve larger and larger arenas, for efficiency, until we + # reach a large-ish size (roughly L3 cache-sized) + if self._size < self._DOUBLE_ARENA_SIZE_UNTIL: + self._size *= 2 + util.info('allocating a new mmap of length %d', length) + arena = Arena(length) + self._arenas.append(arena) + return (arena, 0, length) + + def _discard_arena(self, arena): + # Possibly delete the given (unused) arena + length = arena.size + # Reusing an existing arena is faster than creating a new one, so + # we only reclaim space if it's large enough. + if length < self._DISCARD_FREE_SPACE_LARGER_THAN: + return + blocks = self._allocated_blocks.pop(arena) + assert not blocks + del self._start_to_block[(arena, 0)] + del self._stop_to_block[(arena, length)] + self._arenas.remove(arena) + seq = self._len_to_seq[length] + seq.remove((arena, 0, length)) + if not seq: + del self._len_to_seq[length] + self._lengths.remove(length) + + def _malloc(self, size): + # returns a large enough block -- it might be much larger + i = bisect.bisect_left(self._lengths, size) + if i == len(self._lengths): + return self._new_arena(size) + else: + length = self._lengths[i] + seq = self._len_to_seq[length] + block = seq.pop() + if not seq: + del self._len_to_seq[length], self._lengths[i] + + (arena, start, stop) = block + del self._start_to_block[(arena, start)] + del self._stop_to_block[(arena, stop)] + return block + + def _add_free_block(self, block): + # make block available and try to merge with its neighbours in the arena + (arena, start, stop) = block + + try: + prev_block = self._stop_to_block[(arena, start)] + except KeyError: + pass + else: + start, _ = self._absorb(prev_block) + + try: + next_block = self._start_to_block[(arena, stop)] + except KeyError: + pass + else: + _, stop = self._absorb(next_block) + + block = (arena, start, stop) + length = stop - start + + try: + self._len_to_seq[length].append(block) + except KeyError: + self._len_to_seq[length] = [block] + bisect.insort(self._lengths, length) + + self._start_to_block[(arena, start)] = block + self._stop_to_block[(arena, stop)] = block + + def _absorb(self, block): + # deregister this block so it can be merged with a neighbour + (arena, start, stop) = block + del self._start_to_block[(arena, start)] + del self._stop_to_block[(arena, stop)] + + length = stop - start + seq = self._len_to_seq[length] + seq.remove(block) + if not seq: + del self._len_to_seq[length] + self._lengths.remove(length) + + return start, stop + + def _remove_allocated_block(self, block): + arena, start, stop = block + blocks = self._allocated_blocks[arena] + blocks.remove((start, stop)) + if not blocks: + # Arena is entirely free, discard it from this process + self._discard_arena(arena) + + def _free_pending_blocks(self): + # Free all the blocks in the pending list - called with the lock held. + while True: + try: + block = self._pending_free_blocks.pop() + except IndexError: + break + self._add_free_block(block) + self._remove_allocated_block(block) + + def free(self, block): + # free a block returned by malloc() + # Since free() can be called asynchronously by the GC, it could happen + # that it's called while self._lock is held: in that case, + # self._lock.acquire() would deadlock (issue #12352). To avoid that, a + # trylock is used instead, and if the lock can't be acquired + # immediately, the block is added to a list of blocks to be freed + # synchronously sometimes later from malloc() or free(), by calling + # _free_pending_blocks() (appending and retrieving from a list is not + # strictly thread-safe but under CPython it's atomic thanks to the GIL). + if os.getpid() != self._lastpid: + raise ValueError( + "My pid ({0:n}) is not last pid {1:n}".format( + os.getpid(),self._lastpid)) + if not self._lock.acquire(False): + # can't acquire the lock right now, add the block to the list of + # pending blocks to free + self._pending_free_blocks.append(block) + else: + # we hold the lock + try: + self._n_frees += 1 + self._free_pending_blocks() + self._add_free_block(block) + self._remove_allocated_block(block) + finally: + self._lock.release() + + def malloc(self, size): + # return a block of right size (possibly rounded up) + if size < 0: + raise ValueError("Size {0:n} out of range".format(size)) + if sys.maxsize <= size: + raise OverflowError("Size {0:n} too large".format(size)) + if os.getpid() != self._lastpid: + self.__init__() # reinitialize after fork + with self._lock: + self._n_mallocs += 1 + # allow pending blocks to be marked available + self._free_pending_blocks() + size = self._roundup(max(size, 1), self._alignment) + (arena, start, stop) = self._malloc(size) + real_stop = start + size + if real_stop < stop: + # if the returned block is larger than necessary, mark + # the remainder available + self._add_free_block((arena, real_stop, stop)) + self._allocated_blocks[arena].add((start, real_stop)) + return (arena, start, real_stop) + +# +# Class wrapping a block allocated out of a Heap -- can be inherited by child process +# + +class BufferWrapper(object): + + _heap = Heap() + + def __init__(self, size): + if size < 0: + raise ValueError("Size {0:n} out of range".format(size)) + if sys.maxsize <= size: + raise OverflowError("Size {0:n} too large".format(size)) + block = BufferWrapper._heap.malloc(size) + self._state = (block, size) + util.Finalize(self, BufferWrapper._heap.free, args=(block,)) + + def create_memoryview(self): + (arena, start, stop), size = self._state + return memoryview(arena.buffer)[start:start+size] diff --git a/lib/python3.1/multiprocessing/managers.py b/lib/python3.1/multiprocessing/managers.py new file mode 100644 index 0000000000000000000000000000000000000000..22292c78b7b5b3a253c1d231ad12b225acd6f390 --- /dev/null +++ b/lib/python3.1/multiprocessing/managers.py @@ -0,0 +1,1378 @@ +# +# Module providing manager classes for dealing +# with shared objects +# +# multiprocessing/managers.py +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +__all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ] + +# +# Imports +# + +import sys +import threading +import signal +import array +import queue +import time +import types +import os +from os import getpid + +from traceback import format_exc + +from . import connection +from .context import reduction, get_spawning_popen, ProcessError +from . import pool +from . import process +from . import util +from . import get_context +try: + from . import shared_memory +except ImportError: + HAS_SHMEM = False +else: + HAS_SHMEM = True + __all__.append('SharedMemoryManager') + +# +# Register some things for pickling +# + +def reduce_array(a): + return array.array, (a.typecode, a.tobytes()) +reduction.register(array.array, reduce_array) + +view_types = [type(getattr({}, name)()) for name in ('items','keys','values')] +if view_types[0] is not list: # only needed in Py3.0 + def rebuild_as_list(obj): + return list, (list(obj),) + for view_type in view_types: + reduction.register(view_type, rebuild_as_list) + +# +# Type for identifying shared objects +# + +class Token(object): + ''' + Type to uniquely identify a shared object + ''' + __slots__ = ('typeid', 'address', 'id') + + def __init__(self, typeid, address, id): + (self.typeid, self.address, self.id) = (typeid, address, id) + + def __getstate__(self): + return (self.typeid, self.address, self.id) + + def __setstate__(self, state): + (self.typeid, self.address, self.id) = state + + def __repr__(self): + return '%s(typeid=%r, address=%r, id=%r)' % \ + (self.__class__.__name__, self.typeid, self.address, self.id) + +# +# Function for communication with a manager's server process +# + +def dispatch(c, id, methodname, args=(), kwds={}): + ''' + Send a message to manager using connection `c` and return response + ''' + c.send((id, methodname, args, kwds)) + kind, result = c.recv() + if kind == '#RETURN': + return result + raise convert_to_error(kind, result) + +def convert_to_error(kind, result): + if kind == '#ERROR': + return result + elif kind in ('#TRACEBACK', '#UNSERIALIZABLE'): + if not isinstance(result, str): + raise TypeError( + "Result {0!r} (kind '{1}') type is {2}, not str".format( + result, kind, type(result))) + if kind == '#UNSERIALIZABLE': + return RemoteError('Unserializable message: %s\n' % result) + else: + return RemoteError(result) + else: + return ValueError('Unrecognized message type {!r}'.format(kind)) + +class RemoteError(Exception): + def __str__(self): + return ('\n' + '-'*75 + '\n' + str(self.args[0]) + '-'*75) + +# +# Functions for finding the method names of an object +# + +def all_methods(obj): + ''' + Return a list of names of methods of `obj` + ''' + temp = [] + for name in dir(obj): + func = getattr(obj, name) + if callable(func): + temp.append(name) + return temp + +def public_methods(obj): + ''' + Return a list of names of methods of `obj` which do not start with '_' + ''' + return [name for name in all_methods(obj) if name[0] != '_'] + +# +# Server which is run in a process controlled by a manager +# + +class Server(object): + ''' + Server class which runs in a process controlled by a manager object + ''' + public = ['shutdown', 'create', 'accept_connection', 'get_methods', + 'debug_info', 'number_of_objects', 'dummy', 'incref', 'decref'] + + def __init__(self, registry, address, authkey, serializer): + if not isinstance(authkey, bytes): + raise TypeError( + "Authkey {0!r} is type {1!s}, not bytes".format( + authkey, type(authkey))) + self.registry = registry + self.authkey = process.AuthenticationString(authkey) + Listener, Client = listener_client[serializer] + + # do authentication later + self.listener = Listener(address=address, backlog=16) + self.address = self.listener.address + + self.id_to_obj = {'0': (None, ())} + self.id_to_refcount = {} + self.id_to_local_proxy_obj = {} + self.mutex = threading.Lock() + + def serve_forever(self): + ''' + Run the server forever + ''' + self.stop_event = threading.Event() + process.current_process()._manager_server = self + try: + accepter = threading.Thread(target=self.accepter) + accepter.daemon = True + accepter.start() + try: + while not self.stop_event.is_set(): + self.stop_event.wait(1) + except (KeyboardInterrupt, SystemExit): + pass + finally: + if sys.stdout != sys.__stdout__: # what about stderr? + util.debug('resetting stdout, stderr') + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + sys.exit(0) + + def accepter(self): + while True: + try: + c = self.listener.accept() + except OSError: + continue + t = threading.Thread(target=self.handle_request, args=(c,)) + t.daemon = True + t.start() + + def _handle_request(self, c): + request = None + try: + connection.deliver_challenge(c, self.authkey) + connection.answer_challenge(c, self.authkey) + request = c.recv() + ignore, funcname, args, kwds = request + assert funcname in self.public, '%r unrecognized' % funcname + func = getattr(self, funcname) + except Exception: + msg = ('#TRACEBACK', format_exc()) + else: + try: + result = func(c, *args, **kwds) + except Exception: + msg = ('#TRACEBACK', format_exc()) + else: + msg = ('#RETURN', result) + + try: + c.send(msg) + except Exception as e: + try: + c.send(('#TRACEBACK', format_exc())) + except Exception: + pass + util.info('Failure to send message: %r', msg) + util.info(' ... request was %r', request) + util.info(' ... exception was %r', e) + + def handle_request(self, conn): + ''' + Handle a new connection + ''' + try: + self._handle_request(conn) + except SystemExit: + # Server.serve_client() calls sys.exit(0) on EOF + pass + finally: + conn.close() + + def serve_client(self, conn): + ''' + Handle requests from the proxies in a particular process/thread + ''' + util.debug('starting server thread to service %r', + threading.current_thread().name) + + recv = conn.recv + send = conn.send + id_to_obj = self.id_to_obj + + while not self.stop_event.is_set(): + + try: + methodname = obj = None + request = recv() + ident, methodname, args, kwds = request + try: + obj, exposed, gettypeid = id_to_obj[ident] + except KeyError as ke: + try: + obj, exposed, gettypeid = \ + self.id_to_local_proxy_obj[ident] + except KeyError: + raise ke + + if methodname not in exposed: + raise AttributeError( + 'method %r of %r object is not in exposed=%r' % + (methodname, type(obj), exposed) + ) + + function = getattr(obj, methodname) + + try: + res = function(*args, **kwds) + except Exception as e: + msg = ('#ERROR', e) + else: + typeid = gettypeid and gettypeid.get(methodname, None) + if typeid: + rident, rexposed = self.create(conn, typeid, res) + token = Token(typeid, self.address, rident) + msg = ('#PROXY', (rexposed, token)) + else: + msg = ('#RETURN', res) + + except AttributeError: + if methodname is None: + msg = ('#TRACEBACK', format_exc()) + else: + try: + fallback_func = self.fallback_mapping[methodname] + result = fallback_func( + self, conn, ident, obj, *args, **kwds + ) + msg = ('#RETURN', result) + except Exception: + msg = ('#TRACEBACK', format_exc()) + + except EOFError: + util.debug('got EOF -- exiting thread serving %r', + threading.current_thread().name) + sys.exit(0) + + except Exception: + msg = ('#TRACEBACK', format_exc()) + + try: + try: + send(msg) + except Exception: + send(('#UNSERIALIZABLE', format_exc())) + except Exception as e: + util.info('exception in thread serving %r', + threading.current_thread().name) + util.info(' ... message was %r', msg) + util.info(' ... exception was %r', e) + conn.close() + sys.exit(1) + + def fallback_getvalue(self, conn, ident, obj): + return obj + + def fallback_str(self, conn, ident, obj): + return str(obj) + + def fallback_repr(self, conn, ident, obj): + return repr(obj) + + fallback_mapping = { + '__str__':fallback_str, + '__repr__':fallback_repr, + '#GETVALUE':fallback_getvalue + } + + def dummy(self, c): + pass + + def debug_info(self, c): + ''' + Return some info --- useful to spot problems with refcounting + ''' + # Perhaps include debug info about 'c'? + with self.mutex: + result = [] + keys = list(self.id_to_refcount.keys()) + keys.sort() + for ident in keys: + if ident != '0': + result.append(' %s: refcount=%s\n %s' % + (ident, self.id_to_refcount[ident], + str(self.id_to_obj[ident][0])[:75])) + return '\n'.join(result) + + def number_of_objects(self, c): + ''' + Number of shared objects + ''' + # Doesn't use (len(self.id_to_obj) - 1) as we shouldn't count ident='0' + return len(self.id_to_refcount) + + def shutdown(self, c): + ''' + Shutdown this process + ''' + try: + util.debug('manager received shutdown message') + c.send(('#RETURN', None)) + except: + import traceback + traceback.print_exc() + finally: + self.stop_event.set() + + def create(self, c, typeid, /, *args, **kwds): + ''' + Create a new shared object and return its id + ''' + with self.mutex: + callable, exposed, method_to_typeid, proxytype = \ + self.registry[typeid] + + if callable is None: + if kwds or (len(args) != 1): + raise ValueError( + "Without callable, must have one non-keyword argument") + obj = args[0] + else: + obj = callable(*args, **kwds) + + if exposed is None: + exposed = public_methods(obj) + if method_to_typeid is not None: + if not isinstance(method_to_typeid, dict): + raise TypeError( + "Method_to_typeid {0!r}: type {1!s}, not dict".format( + method_to_typeid, type(method_to_typeid))) + exposed = list(exposed) + list(method_to_typeid) + + ident = '%x' % id(obj) # convert to string because xmlrpclib + # only has 32 bit signed integers + util.debug('%r callable returned object with id %r', typeid, ident) + + self.id_to_obj[ident] = (obj, set(exposed), method_to_typeid) + if ident not in self.id_to_refcount: + self.id_to_refcount[ident] = 0 + + self.incref(c, ident) + return ident, tuple(exposed) + + def get_methods(self, c, token): + ''' + Return the methods of the shared object indicated by token + ''' + return tuple(self.id_to_obj[token.id][1]) + + def accept_connection(self, c, name): + ''' + Spawn a new thread to serve this connection + ''' + threading.current_thread().name = name + c.send(('#RETURN', None)) + self.serve_client(c) + + def incref(self, c, ident): + with self.mutex: + try: + self.id_to_refcount[ident] += 1 + except KeyError as ke: + # If no external references exist but an internal (to the + # manager) still does and a new external reference is created + # from it, restore the manager's tracking of it from the + # previously stashed internal ref. + if ident in self.id_to_local_proxy_obj: + self.id_to_refcount[ident] = 1 + self.id_to_obj[ident] = \ + self.id_to_local_proxy_obj[ident] + obj, exposed, gettypeid = self.id_to_obj[ident] + util.debug('Server re-enabled tracking & INCREF %r', ident) + else: + raise ke + + def decref(self, c, ident): + if ident not in self.id_to_refcount and \ + ident in self.id_to_local_proxy_obj: + util.debug('Server DECREF skipping %r', ident) + return + + with self.mutex: + if self.id_to_refcount[ident] <= 0: + raise AssertionError( + "Id {0!s} ({1!r}) has refcount {2:n}, not 1+".format( + ident, self.id_to_obj[ident], + self.id_to_refcount[ident])) + self.id_to_refcount[ident] -= 1 + if self.id_to_refcount[ident] == 0: + del self.id_to_refcount[ident] + + if ident not in self.id_to_refcount: + # Two-step process in case the object turns out to contain other + # proxy objects (e.g. a managed list of managed lists). + # Otherwise, deleting self.id_to_obj[ident] would trigger the + # deleting of the stored value (another managed object) which would + # in turn attempt to acquire the mutex that is already held here. + self.id_to_obj[ident] = (None, (), None) # thread-safe + util.debug('disposing of obj with id %r', ident) + with self.mutex: + del self.id_to_obj[ident] + + +# +# Class to represent state of a manager +# + +class State(object): + __slots__ = ['value'] + INITIAL = 0 + STARTED = 1 + SHUTDOWN = 2 + +# +# Mapping from serializer name to Listener and Client types +# + +listener_client = { + 'pickle' : (connection.Listener, connection.Client), + 'xmlrpclib' : (connection.XmlListener, connection.XmlClient) + } + +# +# Definition of BaseManager +# + +class BaseManager(object): + ''' + Base class for managers + ''' + _registry = {} + _Server = Server + + def __init__(self, address=None, authkey=None, serializer='pickle', + ctx=None): + if authkey is None: + authkey = process.current_process().authkey + self._address = address # XXX not final address if eg ('', 0) + self._authkey = process.AuthenticationString(authkey) + self._state = State() + self._state.value = State.INITIAL + self._serializer = serializer + self._Listener, self._Client = listener_client[serializer] + self._ctx = ctx or get_context() + + def get_server(self): + ''' + Return server object with serve_forever() method and address attribute + ''' + if self._state.value != State.INITIAL: + if self._state.value == State.STARTED: + raise ProcessError("Already started server") + elif self._state.value == State.SHUTDOWN: + raise ProcessError("Manager has shut down") + else: + raise ProcessError( + "Unknown state {!r}".format(self._state.value)) + return Server(self._registry, self._address, + self._authkey, self._serializer) + + def connect(self): + ''' + Connect manager object to the server process + ''' + Listener, Client = listener_client[self._serializer] + conn = Client(self._address, authkey=self._authkey) + dispatch(conn, None, 'dummy') + self._state.value = State.STARTED + + def start(self, initializer=None, initargs=()): + ''' + Spawn a server process for this manager object + ''' + if self._state.value != State.INITIAL: + if self._state.value == State.STARTED: + raise ProcessError("Already started server") + elif self._state.value == State.SHUTDOWN: + raise ProcessError("Manager has shut down") + else: + raise ProcessError( + "Unknown state {!r}".format(self._state.value)) + + if initializer is not None and not callable(initializer): + raise TypeError('initializer must be a callable') + + # pipe over which we will retrieve address of server + reader, writer = connection.Pipe(duplex=False) + + # spawn process which runs a server + self._process = self._ctx.Process( + target=type(self)._run_server, + args=(self._registry, self._address, self._authkey, + self._serializer, writer, initializer, initargs), + ) + ident = ':'.join(str(i) for i in self._process._identity) + self._process.name = type(self).__name__ + '-' + ident + self._process.start() + + # get address of server + writer.close() + self._address = reader.recv() + reader.close() + + # register a finalizer + self._state.value = State.STARTED + self.shutdown = util.Finalize( + self, type(self)._finalize_manager, + args=(self._process, self._address, self._authkey, + self._state, self._Client), + exitpriority=0 + ) + + @classmethod + def _run_server(cls, registry, address, authkey, serializer, writer, + initializer=None, initargs=()): + ''' + Create a server, report its address and run it + ''' + # bpo-36368: protect server process from KeyboardInterrupt signals + signal.signal(signal.SIGINT, signal.SIG_IGN) + + if initializer is not None: + initializer(*initargs) + + # create server + server = cls._Server(registry, address, authkey, serializer) + + # inform parent process of the server's address + writer.send(server.address) + writer.close() + + # run the manager + util.info('manager serving at %r', server.address) + server.serve_forever() + + def _create(self, typeid, /, *args, **kwds): + ''' + Create a new shared object; return the token and exposed tuple + ''' + assert self._state.value == State.STARTED, 'server not yet started' + conn = self._Client(self._address, authkey=self._authkey) + try: + id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds) + finally: + conn.close() + return Token(typeid, self._address, id), exposed + + def join(self, timeout=None): + ''' + Join the manager process (if it has been spawned) + ''' + if self._process is not None: + self._process.join(timeout) + if not self._process.is_alive(): + self._process = None + + def _debug_info(self): + ''' + Return some info about the servers shared objects and connections + ''' + conn = self._Client(self._address, authkey=self._authkey) + try: + return dispatch(conn, None, 'debug_info') + finally: + conn.close() + + def _number_of_objects(self): + ''' + Return the number of shared objects + ''' + conn = self._Client(self._address, authkey=self._authkey) + try: + return dispatch(conn, None, 'number_of_objects') + finally: + conn.close() + + def __enter__(self): + if self._state.value == State.INITIAL: + self.start() + if self._state.value != State.STARTED: + if self._state.value == State.INITIAL: + raise ProcessError("Unable to start server") + elif self._state.value == State.SHUTDOWN: + raise ProcessError("Manager has shut down") + else: + raise ProcessError( + "Unknown state {!r}".format(self._state.value)) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.shutdown() + + @staticmethod + def _finalize_manager(process, address, authkey, state, _Client): + ''' + Shutdown the manager process; will be registered as a finalizer + ''' + if process.is_alive(): + util.info('sending shutdown message to manager') + try: + conn = _Client(address, authkey=authkey) + try: + dispatch(conn, None, 'shutdown') + finally: + conn.close() + except Exception: + pass + + process.join(timeout=1.0) + if process.is_alive(): + util.info('manager still alive') + if hasattr(process, 'terminate'): + util.info('trying to `terminate()` manager process') + process.terminate() + process.join(timeout=1.0) + if process.is_alive(): + util.info('manager still alive after terminate') + + state.value = State.SHUTDOWN + try: + del BaseProxy._address_to_local[address] + except KeyError: + pass + + @property + def address(self): + return self._address + + @classmethod + def register(cls, typeid, callable=None, proxytype=None, exposed=None, + method_to_typeid=None, create_method=True): + ''' + Register a typeid with the manager type + ''' + if '_registry' not in cls.__dict__: + cls._registry = cls._registry.copy() + + if proxytype is None: + proxytype = AutoProxy + + exposed = exposed or getattr(proxytype, '_exposed_', None) + + method_to_typeid = method_to_typeid or \ + getattr(proxytype, '_method_to_typeid_', None) + + if method_to_typeid: + for key, value in list(method_to_typeid.items()): # isinstance? + assert type(key) is str, '%r is not a string' % key + assert type(value) is str, '%r is not a string' % value + + cls._registry[typeid] = ( + callable, exposed, method_to_typeid, proxytype + ) + + if create_method: + def temp(self, /, *args, **kwds): + util.debug('requesting creation of a shared %r object', typeid) + token, exp = self._create(typeid, *args, **kwds) + proxy = proxytype( + token, self._serializer, manager=self, + authkey=self._authkey, exposed=exp + ) + conn = self._Client(token.address, authkey=self._authkey) + dispatch(conn, None, 'decref', (token.id,)) + return proxy + temp.__name__ = typeid + setattr(cls, typeid, temp) + +# +# Subclass of set which get cleared after a fork +# + +class ProcessLocalSet(set): + def __init__(self): + util.register_after_fork(self, lambda obj: obj.clear()) + def __reduce__(self): + return type(self), () + +# +# Definition of BaseProxy +# + +class BaseProxy(object): + ''' + A base for proxies of shared objects + ''' + _address_to_local = {} + _mutex = util.ForkAwareThreadLock() + + def __init__(self, token, serializer, manager=None, + authkey=None, exposed=None, incref=True, manager_owned=False): + with BaseProxy._mutex: + tls_idset = BaseProxy._address_to_local.get(token.address, None) + if tls_idset is None: + tls_idset = util.ForkAwareLocal(), ProcessLocalSet() + BaseProxy._address_to_local[token.address] = tls_idset + + # self._tls is used to record the connection used by this + # thread to communicate with the manager at token.address + self._tls = tls_idset[0] + + # self._idset is used to record the identities of all shared + # objects for which the current process owns references and + # which are in the manager at token.address + self._idset = tls_idset[1] + + self._token = token + self._id = self._token.id + self._manager = manager + self._serializer = serializer + self._Client = listener_client[serializer][1] + + # Should be set to True only when a proxy object is being created + # on the manager server; primary use case: nested proxy objects. + # RebuildProxy detects when a proxy is being created on the manager + # and sets this value appropriately. + self._owned_by_manager = manager_owned + + if authkey is not None: + self._authkey = process.AuthenticationString(authkey) + elif self._manager is not None: + self._authkey = self._manager._authkey + else: + self._authkey = process.current_process().authkey + + if incref: + self._incref() + + util.register_after_fork(self, BaseProxy._after_fork) + + def _connect(self): + util.debug('making connection to manager') + name = process.current_process().name + if threading.current_thread().name != 'MainThread': + name += '|' + threading.current_thread().name + conn = self._Client(self._token.address, authkey=self._authkey) + dispatch(conn, None, 'accept_connection', (name,)) + self._tls.connection = conn + + def _callmethod(self, methodname, args=(), kwds={}): + ''' + Try to call a method of the referent and return a copy of the result + ''' + try: + conn = self._tls.connection + except AttributeError: + util.debug('thread %r does not own a connection', + threading.current_thread().name) + self._connect() + conn = self._tls.connection + + conn.send((self._id, methodname, args, kwds)) + kind, result = conn.recv() + + if kind == '#RETURN': + return result + elif kind == '#PROXY': + exposed, token = result + proxytype = self._manager._registry[token.typeid][-1] + token.address = self._token.address + proxy = proxytype( + token, self._serializer, manager=self._manager, + authkey=self._authkey, exposed=exposed + ) + conn = self._Client(token.address, authkey=self._authkey) + dispatch(conn, None, 'decref', (token.id,)) + return proxy + raise convert_to_error(kind, result) + + def _getvalue(self): + ''' + Get a copy of the value of the referent + ''' + return self._callmethod('#GETVALUE') + + def _incref(self): + if self._owned_by_manager: + util.debug('owned_by_manager skipped INCREF of %r', self._token.id) + return + + conn = self._Client(self._token.address, authkey=self._authkey) + dispatch(conn, None, 'incref', (self._id,)) + util.debug('INCREF %r', self._token.id) + + self._idset.add(self._id) + + state = self._manager and self._manager._state + + self._close = util.Finalize( + self, BaseProxy._decref, + args=(self._token, self._authkey, state, + self._tls, self._idset, self._Client), + exitpriority=10 + ) + + @staticmethod + def _decref(token, authkey, state, tls, idset, _Client): + idset.discard(token.id) + + # check whether manager is still alive + if state is None or state.value == State.STARTED: + # tell manager this process no longer cares about referent + try: + util.debug('DECREF %r', token.id) + conn = _Client(token.address, authkey=authkey) + dispatch(conn, None, 'decref', (token.id,)) + except Exception as e: + util.debug('... decref failed %s', e) + + else: + util.debug('DECREF %r -- manager already shutdown', token.id) + + # check whether we can close this thread's connection because + # the process owns no more references to objects for this manager + if not idset and hasattr(tls, 'connection'): + util.debug('thread %r has no more proxies so closing conn', + threading.current_thread().name) + tls.connection.close() + del tls.connection + + def _after_fork(self): + self._manager = None + try: + self._incref() + except Exception as e: + # the proxy may just be for a manager which has shutdown + util.info('incref failed: %s' % e) + + def __reduce__(self): + kwds = {} + if get_spawning_popen() is not None: + kwds['authkey'] = self._authkey + + if getattr(self, '_isauto', False): + kwds['exposed'] = self._exposed_ + return (RebuildProxy, + (AutoProxy, self._token, self._serializer, kwds)) + else: + return (RebuildProxy, + (type(self), self._token, self._serializer, kwds)) + + def __deepcopy__(self, memo): + return self._getvalue() + + def __repr__(self): + return '<%s object, typeid %r at %#x>' % \ + (type(self).__name__, self._token.typeid, id(self)) + + def __str__(self): + ''' + Return representation of the referent (or a fall-back if that fails) + ''' + try: + return self._callmethod('__repr__') + except Exception: + return repr(self)[:-1] + "; '__str__()' failed>" + +# +# Function used for unpickling +# + +def RebuildProxy(func, token, serializer, kwds): + ''' + Function used for unpickling proxy objects. + ''' + server = getattr(process.current_process(), '_manager_server', None) + if server and server.address == token.address: + util.debug('Rebuild a proxy owned by manager, token=%r', token) + kwds['manager_owned'] = True + if token.id not in server.id_to_local_proxy_obj: + server.id_to_local_proxy_obj[token.id] = \ + server.id_to_obj[token.id] + incref = ( + kwds.pop('incref', True) and + not getattr(process.current_process(), '_inheriting', False) + ) + return func(token, serializer, incref=incref, **kwds) + +# +# Functions to create proxies and proxy types +# + +def MakeProxyType(name, exposed, _cache={}): + ''' + Return a proxy type whose methods are given by `exposed` + ''' + exposed = tuple(exposed) + try: + return _cache[(name, exposed)] + except KeyError: + pass + + dic = {} + + for meth in exposed: + exec('''def %s(self, /, *args, **kwds): + return self._callmethod(%r, args, kwds)''' % (meth, meth), dic) + + ProxyType = type(name, (BaseProxy,), dic) + ProxyType._exposed_ = exposed + _cache[(name, exposed)] = ProxyType + return ProxyType + + +def AutoProxy(token, serializer, manager=None, authkey=None, + exposed=None, incref=True, manager_owned=False): + ''' + Return an auto-proxy for `token` + ''' + _Client = listener_client[serializer][1] + + if exposed is None: + conn = _Client(token.address, authkey=authkey) + try: + exposed = dispatch(conn, None, 'get_methods', (token,)) + finally: + conn.close() + + if authkey is None and manager is not None: + authkey = manager._authkey + if authkey is None: + authkey = process.current_process().authkey + + ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed) + proxy = ProxyType(token, serializer, manager=manager, authkey=authkey, + incref=incref, manager_owned=manager_owned) + proxy._isauto = True + return proxy + +# +# Types/callables which we will register with SyncManager +# + +class Namespace(object): + def __init__(self, /, **kwds): + self.__dict__.update(kwds) + def __repr__(self): + items = list(self.__dict__.items()) + temp = [] + for name, value in items: + if not name.startswith('_'): + temp.append('%s=%r' % (name, value)) + temp.sort() + return '%s(%s)' % (self.__class__.__name__, ', '.join(temp)) + +class Value(object): + def __init__(self, typecode, value, lock=True): + self._typecode = typecode + self._value = value + def get(self): + return self._value + def set(self, value): + self._value = value + def __repr__(self): + return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value) + value = property(get, set) + +def Array(typecode, sequence, lock=True): + return array.array(typecode, sequence) + +# +# Proxy types used by SyncManager +# + +class IteratorProxy(BaseProxy): + _exposed_ = ('__next__', 'send', 'throw', 'close') + def __iter__(self): + return self + def __next__(self, *args): + return self._callmethod('__next__', args) + def send(self, *args): + return self._callmethod('send', args) + def throw(self, *args): + return self._callmethod('throw', args) + def close(self, *args): + return self._callmethod('close', args) + + +class AcquirerProxy(BaseProxy): + _exposed_ = ('acquire', 'release') + def acquire(self, blocking=True, timeout=None): + args = (blocking,) if timeout is None else (blocking, timeout) + return self._callmethod('acquire', args) + def release(self): + return self._callmethod('release') + def __enter__(self): + return self._callmethod('acquire') + def __exit__(self, exc_type, exc_val, exc_tb): + return self._callmethod('release') + + +class ConditionProxy(AcquirerProxy): + _exposed_ = ('acquire', 'release', 'wait', 'notify', 'notify_all') + def wait(self, timeout=None): + return self._callmethod('wait', (timeout,)) + def notify(self, n=1): + return self._callmethod('notify', (n,)) + def notify_all(self): + return self._callmethod('notify_all') + def wait_for(self, predicate, timeout=None): + result = predicate() + if result: + return result + if timeout is not None: + endtime = time.monotonic() + timeout + else: + endtime = None + waittime = None + while not result: + if endtime is not None: + waittime = endtime - time.monotonic() + if waittime <= 0: + break + self.wait(waittime) + result = predicate() + return result + + +class EventProxy(BaseProxy): + _exposed_ = ('is_set', 'set', 'clear', 'wait') + def is_set(self): + return self._callmethod('is_set') + def set(self): + return self._callmethod('set') + def clear(self): + return self._callmethod('clear') + def wait(self, timeout=None): + return self._callmethod('wait', (timeout,)) + + +class BarrierProxy(BaseProxy): + _exposed_ = ('__getattribute__', 'wait', 'abort', 'reset') + def wait(self, timeout=None): + return self._callmethod('wait', (timeout,)) + def abort(self): + return self._callmethod('abort') + def reset(self): + return self._callmethod('reset') + @property + def parties(self): + return self._callmethod('__getattribute__', ('parties',)) + @property + def n_waiting(self): + return self._callmethod('__getattribute__', ('n_waiting',)) + @property + def broken(self): + return self._callmethod('__getattribute__', ('broken',)) + + +class NamespaceProxy(BaseProxy): + _exposed_ = ('__getattribute__', '__setattr__', '__delattr__') + def __getattr__(self, key): + if key[0] == '_': + return object.__getattribute__(self, key) + callmethod = object.__getattribute__(self, '_callmethod') + return callmethod('__getattribute__', (key,)) + def __setattr__(self, key, value): + if key[0] == '_': + return object.__setattr__(self, key, value) + callmethod = object.__getattribute__(self, '_callmethod') + return callmethod('__setattr__', (key, value)) + def __delattr__(self, key): + if key[0] == '_': + return object.__delattr__(self, key) + callmethod = object.__getattribute__(self, '_callmethod') + return callmethod('__delattr__', (key,)) + + +class ValueProxy(BaseProxy): + _exposed_ = ('get', 'set') + def get(self): + return self._callmethod('get') + def set(self, value): + return self._callmethod('set', (value,)) + value = property(get, set) + + __class_getitem__ = classmethod(types.GenericAlias) + + +BaseListProxy = MakeProxyType('BaseListProxy', ( + '__add__', '__contains__', '__delitem__', '__getitem__', '__len__', + '__mul__', '__reversed__', '__rmul__', '__setitem__', + 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', + 'reverse', 'sort', '__imul__' + )) +class ListProxy(BaseListProxy): + def __iadd__(self, value): + self._callmethod('extend', (value,)) + return self + def __imul__(self, value): + self._callmethod('__imul__', (value,)) + return self + + +DictProxy = MakeProxyType('DictProxy', ( + '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__', + '__setitem__', 'clear', 'copy', 'get', 'items', + 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values' + )) +DictProxy._method_to_typeid_ = { + '__iter__': 'Iterator', + } + + +ArrayProxy = MakeProxyType('ArrayProxy', ( + '__len__', '__getitem__', '__setitem__' + )) + + +BasePoolProxy = MakeProxyType('PoolProxy', ( + 'apply', 'apply_async', 'close', 'imap', 'imap_unordered', 'join', + 'map', 'map_async', 'starmap', 'starmap_async', 'terminate', + )) +BasePoolProxy._method_to_typeid_ = { + 'apply_async': 'AsyncResult', + 'map_async': 'AsyncResult', + 'starmap_async': 'AsyncResult', + 'imap': 'Iterator', + 'imap_unordered': 'Iterator' + } +class PoolProxy(BasePoolProxy): + def __enter__(self): + return self + def __exit__(self, exc_type, exc_val, exc_tb): + self.terminate() + +# +# Definition of SyncManager +# + +class SyncManager(BaseManager): + ''' + Subclass of `BaseManager` which supports a number of shared object types. + + The types registered are those intended for the synchronization + of threads, plus `dict`, `list` and `Namespace`. + + The `multiprocessing.Manager()` function creates started instances of + this class. + ''' + +SyncManager.register('Queue', queue.Queue) +SyncManager.register('JoinableQueue', queue.Queue) +SyncManager.register('Event', threading.Event, EventProxy) +SyncManager.register('Lock', threading.Lock, AcquirerProxy) +SyncManager.register('RLock', threading.RLock, AcquirerProxy) +SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy) +SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore, + AcquirerProxy) +SyncManager.register('Condition', threading.Condition, ConditionProxy) +SyncManager.register('Barrier', threading.Barrier, BarrierProxy) +SyncManager.register('Pool', pool.Pool, PoolProxy) +SyncManager.register('list', list, ListProxy) +SyncManager.register('dict', dict, DictProxy) +SyncManager.register('Value', Value, ValueProxy) +SyncManager.register('Array', Array, ArrayProxy) +SyncManager.register('Namespace', Namespace, NamespaceProxy) + +# types returned by methods of PoolProxy +SyncManager.register('Iterator', proxytype=IteratorProxy, create_method=False) +SyncManager.register('AsyncResult', create_method=False) + +# +# Definition of SharedMemoryManager and SharedMemoryServer +# + +if HAS_SHMEM: + class _SharedMemoryTracker: + "Manages one or more shared memory segments." + + def __init__(self, name, segment_names=[]): + self.shared_memory_context_name = name + self.segment_names = segment_names + + def register_segment(self, segment_name): + "Adds the supplied shared memory block name to tracker." + util.debug(f"Register segment {segment_name!r} in pid {getpid()}") + self.segment_names.append(segment_name) + + def destroy_segment(self, segment_name): + """Calls unlink() on the shared memory block with the supplied name + and removes it from the list of blocks being tracked.""" + util.debug(f"Destroy segment {segment_name!r} in pid {getpid()}") + self.segment_names.remove(segment_name) + segment = shared_memory.SharedMemory(segment_name) + segment.close() + segment.unlink() + + def unlink(self): + "Calls destroy_segment() on all tracked shared memory blocks." + for segment_name in self.segment_names[:]: + self.destroy_segment(segment_name) + + def __del__(self): + util.debug(f"Call {self.__class__.__name__}.__del__ in {getpid()}") + self.unlink() + + def __getstate__(self): + return (self.shared_memory_context_name, self.segment_names) + + def __setstate__(self, state): + self.__init__(*state) + + + class SharedMemoryServer(Server): + + public = Server.public + \ + ['track_segment', 'release_segment', 'list_segments'] + + def __init__(self, *args, **kwargs): + Server.__init__(self, *args, **kwargs) + address = self.address + # The address of Linux abstract namespaces can be bytes + if isinstance(address, bytes): + address = os.fsdecode(address) + self.shared_memory_context = \ + _SharedMemoryTracker(f"shm_{address}_{getpid()}") + util.debug(f"SharedMemoryServer started by pid {getpid()}") + + def create(self, c, typeid, /, *args, **kwargs): + """Create a new distributed-shared object (not backed by a shared + memory block) and return its id to be used in a Proxy Object.""" + # Unless set up as a shared proxy, don't make shared_memory_context + # a standard part of kwargs. This makes things easier for supplying + # simple functions. + if hasattr(self.registry[typeid][-1], "_shared_memory_proxy"): + kwargs['shared_memory_context'] = self.shared_memory_context + return Server.create(self, c, typeid, *args, **kwargs) + + def shutdown(self, c): + "Call unlink() on all tracked shared memory, terminate the Server." + self.shared_memory_context.unlink() + return Server.shutdown(self, c) + + def track_segment(self, c, segment_name): + "Adds the supplied shared memory block name to Server's tracker." + self.shared_memory_context.register_segment(segment_name) + + def release_segment(self, c, segment_name): + """Calls unlink() on the shared memory block with the supplied name + and removes it from the tracker instance inside the Server.""" + self.shared_memory_context.destroy_segment(segment_name) + + def list_segments(self, c): + """Returns a list of names of shared memory blocks that the Server + is currently tracking.""" + return self.shared_memory_context.segment_names + + + class SharedMemoryManager(BaseManager): + """Like SyncManager but uses SharedMemoryServer instead of Server. + + It provides methods for creating and returning SharedMemory instances + and for creating a list-like object (ShareableList) backed by shared + memory. It also provides methods that create and return Proxy Objects + that support synchronization across processes (i.e. multi-process-safe + locks and semaphores). + """ + + _Server = SharedMemoryServer + + def __init__(self, *args, **kwargs): + if os.name == "posix": + # bpo-36867: Ensure the resource_tracker is running before + # launching the manager process, so that concurrent + # shared_memory manipulation both in the manager and in the + # current process does not create two resource_tracker + # processes. + from . import resource_tracker + resource_tracker.ensure_running() + BaseManager.__init__(self, *args, **kwargs) + util.debug(f"{self.__class__.__name__} created by pid {getpid()}") + + def __del__(self): + util.debug(f"{self.__class__.__name__}.__del__ by pid {getpid()}") + pass + + def get_server(self): + 'Better than monkeypatching for now; merge into Server ultimately' + if self._state.value != State.INITIAL: + if self._state.value == State.STARTED: + raise ProcessError("Already started SharedMemoryServer") + elif self._state.value == State.SHUTDOWN: + raise ProcessError("SharedMemoryManager has shut down") + else: + raise ProcessError( + "Unknown state {!r}".format(self._state.value)) + return self._Server(self._registry, self._address, + self._authkey, self._serializer) + + def SharedMemory(self, size): + """Returns a new SharedMemory instance with the specified size in + bytes, to be tracked by the manager.""" + with self._Client(self._address, authkey=self._authkey) as conn: + sms = shared_memory.SharedMemory(None, create=True, size=size) + try: + dispatch(conn, None, 'track_segment', (sms.name,)) + except BaseException as e: + sms.unlink() + raise e + return sms + + def ShareableList(self, sequence): + """Returns a new ShareableList instance populated with the values + from the input sequence, to be tracked by the manager.""" + with self._Client(self._address, authkey=self._authkey) as conn: + sl = shared_memory.ShareableList(sequence) + try: + dispatch(conn, None, 'track_segment', (sl.shm.name,)) + except BaseException as e: + sl.shm.unlink() + raise e + return sl diff --git a/lib/python3.1/multiprocessing/pool.py b/lib/python3.1/multiprocessing/pool.py new file mode 100644 index 0000000000000000000000000000000000000000..961d7e5991847af1ea8034a5a37d9159cf4614ae --- /dev/null +++ b/lib/python3.1/multiprocessing/pool.py @@ -0,0 +1,957 @@ +# +# Module providing the `Pool` class for managing a process pool +# +# multiprocessing/pool.py +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +__all__ = ['Pool', 'ThreadPool'] + +# +# Imports +# + +import collections +import itertools +import os +import queue +import threading +import time +import traceback +import types +import warnings + +# If threading is available then ThreadPool should be provided. Therefore +# we avoid top-level imports which are liable to fail on some systems. +from . import util +from . import get_context, TimeoutError +from .connection import wait + +# +# Constants representing the state of a pool +# + +INIT = "INIT" +RUN = "RUN" +CLOSE = "CLOSE" +TERMINATE = "TERMINATE" + +# +# Miscellaneous +# + +job_counter = itertools.count() + +def mapstar(args): + return list(map(*args)) + +def starmapstar(args): + return list(itertools.starmap(args[0], args[1])) + +# +# Hack to embed stringification of remote traceback in local traceback +# + +class RemoteTraceback(Exception): + def __init__(self, tb): + self.tb = tb + def __str__(self): + return self.tb + +class ExceptionWithTraceback: + def __init__(self, exc, tb): + tb = traceback.format_exception(type(exc), exc, tb) + tb = ''.join(tb) + self.exc = exc + self.tb = '\n"""\n%s"""' % tb + def __reduce__(self): + return rebuild_exc, (self.exc, self.tb) + +def rebuild_exc(exc, tb): + exc.__cause__ = RemoteTraceback(tb) + return exc + +# +# Code run by worker processes +# + +class MaybeEncodingError(Exception): + """Wraps possible unpickleable errors, so they can be + safely sent through the socket.""" + + def __init__(self, exc, value): + self.exc = repr(exc) + self.value = repr(value) + super(MaybeEncodingError, self).__init__(self.exc, self.value) + + def __str__(self): + return "Error sending result: '%s'. Reason: '%s'" % (self.value, + self.exc) + + def __repr__(self): + return "<%s: %s>" % (self.__class__.__name__, self) + + +def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, + wrap_exception=False): + if (maxtasks is not None) and not (isinstance(maxtasks, int) + and maxtasks >= 1): + raise AssertionError("Maxtasks {!r} is not valid".format(maxtasks)) + put = outqueue.put + get = inqueue.get + if hasattr(inqueue, '_writer'): + inqueue._writer.close() + outqueue._reader.close() + + if initializer is not None: + initializer(*initargs) + + completed = 0 + while maxtasks is None or (maxtasks and completed < maxtasks): + try: + task = get() + except (EOFError, OSError): + util.debug('worker got EOFError or OSError -- exiting') + break + + if task is None: + util.debug('worker got sentinel -- exiting') + break + + job, i, func, args, kwds = task + try: + result = (True, func(*args, **kwds)) + except Exception as e: + if wrap_exception and func is not _helper_reraises_exception: + e = ExceptionWithTraceback(e, e.__traceback__) + result = (False, e) + try: + put((job, i, result)) + except Exception as e: + wrapped = MaybeEncodingError(e, result[1]) + util.debug("Possible encoding error while sending result: %s" % ( + wrapped)) + put((job, i, (False, wrapped))) + + task = job = result = func = args = kwds = None + completed += 1 + util.debug('worker exiting after %d tasks' % completed) + +def _helper_reraises_exception(ex): + 'Pickle-able helper function for use by _guarded_task_generation.' + raise ex + +# +# Class representing a process pool +# + +class _PoolCache(dict): + """ + Class that implements a cache for the Pool class that will notify + the pool management threads every time the cache is emptied. The + notification is done by the use of a queue that is provided when + instantiating the cache. + """ + def __init__(self, /, *args, notifier=None, **kwds): + self.notifier = notifier + super().__init__(*args, **kwds) + + def __delitem__(self, item): + super().__delitem__(item) + + # Notify that the cache is empty. This is important because the + # pool keeps maintaining workers until the cache gets drained. This + # eliminates a race condition in which a task is finished after the + # the pool's _handle_workers method has enter another iteration of the + # loop. In this situation, the only event that can wake up the pool + # is the cache to be emptied (no more tasks available). + if not self: + self.notifier.put(None) + +class Pool(object): + ''' + Class which supports an async version of applying functions to arguments. + ''' + _wrap_exception = True + + @staticmethod + def Process(ctx, *args, **kwds): + return ctx.Process(*args, **kwds) + + def __init__(self, processes=None, initializer=None, initargs=(), + maxtasksperchild=None, context=None): + # Attributes initialized early to make sure that they exist in + # __del__() if __init__() raises an exception + self._pool = [] + self._state = INIT + + self._ctx = context or get_context() + self._setup_queues() + self._taskqueue = queue.SimpleQueue() + # The _change_notifier queue exist to wake up self._handle_workers() + # when the cache (self._cache) is empty or when there is a change in + # the _state variable of the thread that runs _handle_workers. + self._change_notifier = self._ctx.SimpleQueue() + self._cache = _PoolCache(notifier=self._change_notifier) + self._maxtasksperchild = maxtasksperchild + self._initializer = initializer + self._initargs = initargs + + if processes is None: + processes = os.cpu_count() or 1 + if processes < 1: + raise ValueError("Number of processes must be at least 1") + if maxtasksperchild is not None: + if not isinstance(maxtasksperchild, int) or maxtasksperchild <= 0: + raise ValueError("maxtasksperchild must be a positive int or None") + + if initializer is not None and not callable(initializer): + raise TypeError('initializer must be a callable') + + self._processes = processes + try: + self._repopulate_pool() + except Exception: + for p in self._pool: + if p.exitcode is None: + p.terminate() + for p in self._pool: + p.join() + raise + + sentinels = self._get_sentinels() + + self._worker_handler = threading.Thread( + target=Pool._handle_workers, + args=(self._cache, self._taskqueue, self._ctx, self.Process, + self._processes, self._pool, self._inqueue, self._outqueue, + self._initializer, self._initargs, self._maxtasksperchild, + self._wrap_exception, sentinels, self._change_notifier) + ) + self._worker_handler.daemon = True + self._worker_handler._state = RUN + self._worker_handler.start() + + + self._task_handler = threading.Thread( + target=Pool._handle_tasks, + args=(self._taskqueue, self._quick_put, self._outqueue, + self._pool, self._cache) + ) + self._task_handler.daemon = True + self._task_handler._state = RUN + self._task_handler.start() + + self._result_handler = threading.Thread( + target=Pool._handle_results, + args=(self._outqueue, self._quick_get, self._cache) + ) + self._result_handler.daemon = True + self._result_handler._state = RUN + self._result_handler.start() + + self._terminate = util.Finalize( + self, self._terminate_pool, + args=(self._taskqueue, self._inqueue, self._outqueue, self._pool, + self._change_notifier, self._worker_handler, self._task_handler, + self._result_handler, self._cache), + exitpriority=15 + ) + self._state = RUN + + # Copy globals as function locals to make sure that they are available + # during Python shutdown when the Pool is destroyed. + def __del__(self, _warn=warnings.warn, RUN=RUN): + if self._state == RUN: + _warn(f"unclosed running multiprocessing pool {self!r}", + ResourceWarning, source=self) + if getattr(self, '_change_notifier', None) is not None: + self._change_notifier.put(None) + + def __repr__(self): + cls = self.__class__ + return (f'<{cls.__module__}.{cls.__qualname__} ' + f'state={self._state} ' + f'pool_size={len(self._pool)}>') + + def _get_sentinels(self): + task_queue_sentinels = [self._outqueue._reader] + self_notifier_sentinels = [self._change_notifier._reader] + return [*task_queue_sentinels, *self_notifier_sentinels] + + @staticmethod + def _get_worker_sentinels(workers): + return [worker.sentinel for worker in + workers if hasattr(worker, "sentinel")] + + @staticmethod + def _join_exited_workers(pool): + """Cleanup after any worker processes which have exited due to reaching + their specified lifetime. Returns True if any workers were cleaned up. + """ + cleaned = False + for i in reversed(range(len(pool))): + worker = pool[i] + if worker.exitcode is not None: + # worker exited + util.debug('cleaning up worker %d' % i) + worker.join() + cleaned = True + del pool[i] + return cleaned + + def _repopulate_pool(self): + return self._repopulate_pool_static(self._ctx, self.Process, + self._processes, + self._pool, self._inqueue, + self._outqueue, self._initializer, + self._initargs, + self._maxtasksperchild, + self._wrap_exception) + + @staticmethod + def _repopulate_pool_static(ctx, Process, processes, pool, inqueue, + outqueue, initializer, initargs, + maxtasksperchild, wrap_exception): + """Bring the number of pool processes up to the specified number, + for use after reaping workers which have exited. + """ + for i in range(processes - len(pool)): + w = Process(ctx, target=worker, + args=(inqueue, outqueue, + initializer, + initargs, maxtasksperchild, + wrap_exception)) + w.name = w.name.replace('Process', 'PoolWorker') + w.daemon = True + w.start() + pool.append(w) + util.debug('added worker') + + @staticmethod + def _maintain_pool(ctx, Process, processes, pool, inqueue, outqueue, + initializer, initargs, maxtasksperchild, + wrap_exception): + """Clean up any exited workers and start replacements for them. + """ + if Pool._join_exited_workers(pool): + Pool._repopulate_pool_static(ctx, Process, processes, pool, + inqueue, outqueue, initializer, + initargs, maxtasksperchild, + wrap_exception) + + def _setup_queues(self): + self._inqueue = self._ctx.SimpleQueue() + self._outqueue = self._ctx.SimpleQueue() + self._quick_put = self._inqueue._writer.send + self._quick_get = self._outqueue._reader.recv + + def _check_running(self): + if self._state != RUN: + raise ValueError("Pool not running") + + def apply(self, func, args=(), kwds={}): + ''' + Equivalent of `func(*args, **kwds)`. + Pool must be running. + ''' + return self.apply_async(func, args, kwds).get() + + def map(self, func, iterable, chunksize=None): + ''' + Apply `func` to each element in `iterable`, collecting the results + in a list that is returned. + ''' + return self._map_async(func, iterable, mapstar, chunksize).get() + + def starmap(self, func, iterable, chunksize=None): + ''' + Like `map()` method but the elements of the `iterable` are expected to + be iterables as well and will be unpacked as arguments. Hence + `func` and (a, b) becomes func(a, b). + ''' + return self._map_async(func, iterable, starmapstar, chunksize).get() + + def starmap_async(self, func, iterable, chunksize=None, callback=None, + error_callback=None): + ''' + Asynchronous version of `starmap()` method. + ''' + return self._map_async(func, iterable, starmapstar, chunksize, + callback, error_callback) + + def _guarded_task_generation(self, result_job, func, iterable): + '''Provides a generator of tasks for imap and imap_unordered with + appropriate handling for iterables which throw exceptions during + iteration.''' + try: + i = -1 + for i, x in enumerate(iterable): + yield (result_job, i, func, (x,), {}) + except Exception as e: + yield (result_job, i+1, _helper_reraises_exception, (e,), {}) + + def imap(self, func, iterable, chunksize=1): + ''' + Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. + ''' + self._check_running() + if chunksize == 1: + result = IMapIterator(self) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) + return result + else: + if chunksize < 1: + raise ValueError( + "Chunksize must be 1+, not {0:n}".format( + chunksize)) + task_batches = Pool._get_tasks(func, iterable, chunksize) + result = IMapIterator(self) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) + return (item for chunk in result for item in chunk) + + def imap_unordered(self, func, iterable, chunksize=1): + ''' + Like `imap()` method but ordering of results is arbitrary. + ''' + self._check_running() + if chunksize == 1: + result = IMapUnorderedIterator(self) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, func, iterable), + result._set_length + )) + return result + else: + if chunksize < 1: + raise ValueError( + "Chunksize must be 1+, not {0!r}".format(chunksize)) + task_batches = Pool._get_tasks(func, iterable, chunksize) + result = IMapUnorderedIterator(self) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapstar, + task_batches), + result._set_length + )) + return (item for chunk in result for item in chunk) + + def apply_async(self, func, args=(), kwds={}, callback=None, + error_callback=None): + ''' + Asynchronous version of `apply()` method. + ''' + self._check_running() + result = ApplyResult(self, callback, error_callback) + self._taskqueue.put(([(result._job, 0, func, args, kwds)], None)) + return result + + def map_async(self, func, iterable, chunksize=None, callback=None, + error_callback=None): + ''' + Asynchronous version of `map()` method. + ''' + return self._map_async(func, iterable, mapstar, chunksize, callback, + error_callback) + + def _map_async(self, func, iterable, mapper, chunksize=None, callback=None, + error_callback=None): + ''' + Helper function to implement map, starmap and their async counterparts. + ''' + self._check_running() + if not hasattr(iterable, '__len__'): + iterable = list(iterable) + + if chunksize is None: + chunksize, extra = divmod(len(iterable), len(self._pool) * 4) + if extra: + chunksize += 1 + if len(iterable) == 0: + chunksize = 0 + + task_batches = Pool._get_tasks(func, iterable, chunksize) + result = MapResult(self, chunksize, len(iterable), callback, + error_callback=error_callback) + self._taskqueue.put( + ( + self._guarded_task_generation(result._job, + mapper, + task_batches), + None + ) + ) + return result + + @staticmethod + def _wait_for_updates(sentinels, change_notifier, timeout=None): + wait(sentinels, timeout=timeout) + while not change_notifier.empty(): + change_notifier.get() + + @classmethod + def _handle_workers(cls, cache, taskqueue, ctx, Process, processes, + pool, inqueue, outqueue, initializer, initargs, + maxtasksperchild, wrap_exception, sentinels, + change_notifier): + thread = threading.current_thread() + + # Keep maintaining workers until the cache gets drained, unless the pool + # is terminated. + while thread._state == RUN or (cache and thread._state != TERMINATE): + cls._maintain_pool(ctx, Process, processes, pool, inqueue, + outqueue, initializer, initargs, + maxtasksperchild, wrap_exception) + + current_sentinels = [*cls._get_worker_sentinels(pool), *sentinels] + + cls._wait_for_updates(current_sentinels, change_notifier) + # send sentinel to stop workers + taskqueue.put(None) + util.debug('worker handler exiting') + + @staticmethod + def _handle_tasks(taskqueue, put, outqueue, pool, cache): + thread = threading.current_thread() + + for taskseq, set_length in iter(taskqueue.get, None): + task = None + try: + # iterating taskseq cannot fail + for task in taskseq: + if thread._state != RUN: + util.debug('task handler found thread._state != RUN') + break + try: + put(task) + except Exception as e: + job, idx = task[:2] + try: + cache[job]._set(idx, (False, e)) + except KeyError: + pass + else: + if set_length: + util.debug('doing set_length()') + idx = task[1] if task else -1 + set_length(idx + 1) + continue + break + finally: + task = taskseq = job = None + else: + util.debug('task handler got sentinel') + + try: + # tell result handler to finish when cache is empty + util.debug('task handler sending sentinel to result handler') + outqueue.put(None) + + # tell workers there is no more work + util.debug('task handler sending sentinel to workers') + for p in pool: + put(None) + except OSError: + util.debug('task handler got OSError when sending sentinels') + + util.debug('task handler exiting') + + @staticmethod + def _handle_results(outqueue, get, cache): + thread = threading.current_thread() + + while 1: + try: + task = get() + except (OSError, EOFError): + util.debug('result handler got EOFError/OSError -- exiting') + return + + if thread._state != RUN: + assert thread._state == TERMINATE, "Thread not in TERMINATE" + util.debug('result handler found thread._state=TERMINATE') + break + + if task is None: + util.debug('result handler got sentinel') + break + + job, i, obj = task + try: + cache[job]._set(i, obj) + except KeyError: + pass + task = job = obj = None + + while cache and thread._state != TERMINATE: + try: + task = get() + except (OSError, EOFError): + util.debug('result handler got EOFError/OSError -- exiting') + return + + if task is None: + util.debug('result handler ignoring extra sentinel') + continue + job, i, obj = task + try: + cache[job]._set(i, obj) + except KeyError: + pass + task = job = obj = None + + if hasattr(outqueue, '_reader'): + util.debug('ensuring that outqueue is not full') + # If we don't make room available in outqueue then + # attempts to add the sentinel (None) to outqueue may + # block. There is guaranteed to be no more than 2 sentinels. + try: + for i in range(10): + if not outqueue._reader.poll(): + break + get() + except (OSError, EOFError): + pass + + util.debug('result handler exiting: len(cache)=%s, thread._state=%s', + len(cache), thread._state) + + @staticmethod + def _get_tasks(func, it, size): + it = iter(it) + while 1: + x = tuple(itertools.islice(it, size)) + if not x: + return + yield (func, x) + + def __reduce__(self): + raise NotImplementedError( + 'pool objects cannot be passed between processes or pickled' + ) + + def close(self): + util.debug('closing pool') + if self._state == RUN: + self._state = CLOSE + self._worker_handler._state = CLOSE + self._change_notifier.put(None) + + def terminate(self): + util.debug('terminating pool') + self._state = TERMINATE + self._terminate() + + def join(self): + util.debug('joining pool') + if self._state == RUN: + raise ValueError("Pool is still running") + elif self._state not in (CLOSE, TERMINATE): + raise ValueError("In unknown state") + self._worker_handler.join() + self._task_handler.join() + self._result_handler.join() + for p in self._pool: + p.join() + + @staticmethod + def _help_stuff_finish(inqueue, task_handler, size): + # task_handler may be blocked trying to put items on inqueue + util.debug('removing tasks from inqueue until task handler finished') + inqueue._rlock.acquire() + while task_handler.is_alive() and inqueue._reader.poll(): + inqueue._reader.recv() + time.sleep(0) + + @classmethod + def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, change_notifier, + worker_handler, task_handler, result_handler, cache): + # this is guaranteed to only be called once + util.debug('finalizing pool') + + # Notify that the worker_handler state has been changed so the + # _handle_workers loop can be unblocked (and exited) in order to + # send the finalization sentinel all the workers. + worker_handler._state = TERMINATE + change_notifier.put(None) + + task_handler._state = TERMINATE + + util.debug('helping task handler/workers to finish') + cls._help_stuff_finish(inqueue, task_handler, len(pool)) + + if (not result_handler.is_alive()) and (len(cache) != 0): + raise AssertionError( + "Cannot have cache with result_hander not alive") + + result_handler._state = TERMINATE + change_notifier.put(None) + outqueue.put(None) # sentinel + + # We must wait for the worker handler to exit before terminating + # workers because we don't want workers to be restarted behind our back. + util.debug('joining worker handler') + if threading.current_thread() is not worker_handler: + worker_handler.join() + + # Terminate workers which haven't already finished. + if pool and hasattr(pool[0], 'terminate'): + util.debug('terminating workers') + for p in pool: + if p.exitcode is None: + p.terminate() + + util.debug('joining task handler') + if threading.current_thread() is not task_handler: + task_handler.join() + + util.debug('joining result handler') + if threading.current_thread() is not result_handler: + result_handler.join() + + if pool and hasattr(pool[0], 'terminate'): + util.debug('joining pool workers') + for p in pool: + if p.is_alive(): + # worker has not yet exited + util.debug('cleaning up worker %d' % p.pid) + p.join() + + def __enter__(self): + self._check_running() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.terminate() + +# +# Class whose instances are returned by `Pool.apply_async()` +# + +class ApplyResult(object): + + def __init__(self, pool, callback, error_callback): + self._pool = pool + self._event = threading.Event() + self._job = next(job_counter) + self._cache = pool._cache + self._callback = callback + self._error_callback = error_callback + self._cache[self._job] = self + + def ready(self): + return self._event.is_set() + + def successful(self): + if not self.ready(): + raise ValueError("{0!r} not ready".format(self)) + return self._success + + def wait(self, timeout=None): + self._event.wait(timeout) + + def get(self, timeout=None): + self.wait(timeout) + if not self.ready(): + raise TimeoutError + if self._success: + return self._value + else: + raise self._value + + def _set(self, i, obj): + self._success, self._value = obj + if self._callback and self._success: + self._callback(self._value) + if self._error_callback and not self._success: + self._error_callback(self._value) + self._event.set() + del self._cache[self._job] + self._pool = None + + __class_getitem__ = classmethod(types.GenericAlias) + +AsyncResult = ApplyResult # create alias -- see #17805 + +# +# Class whose instances are returned by `Pool.map_async()` +# + +class MapResult(ApplyResult): + + def __init__(self, pool, chunksize, length, callback, error_callback): + ApplyResult.__init__(self, pool, callback, + error_callback=error_callback) + self._success = True + self._value = [None] * length + self._chunksize = chunksize + if chunksize <= 0: + self._number_left = 0 + self._event.set() + del self._cache[self._job] + else: + self._number_left = length//chunksize + bool(length % chunksize) + + def _set(self, i, success_result): + self._number_left -= 1 + success, result = success_result + if success and self._success: + self._value[i*self._chunksize:(i+1)*self._chunksize] = result + if self._number_left == 0: + if self._callback: + self._callback(self._value) + del self._cache[self._job] + self._event.set() + self._pool = None + else: + if not success and self._success: + # only store first exception + self._success = False + self._value = result + if self._number_left == 0: + # only consider the result ready once all jobs are done + if self._error_callback: + self._error_callback(self._value) + del self._cache[self._job] + self._event.set() + self._pool = None + +# +# Class whose instances are returned by `Pool.imap()` +# + +class IMapIterator(object): + + def __init__(self, pool): + self._pool = pool + self._cond = threading.Condition(threading.Lock()) + self._job = next(job_counter) + self._cache = pool._cache + self._items = collections.deque() + self._index = 0 + self._length = None + self._unsorted = {} + self._cache[self._job] = self + + def __iter__(self): + return self + + def next(self, timeout=None): + with self._cond: + try: + item = self._items.popleft() + except IndexError: + if self._index == self._length: + self._pool = None + raise StopIteration from None + self._cond.wait(timeout) + try: + item = self._items.popleft() + except IndexError: + if self._index == self._length: + self._pool = None + raise StopIteration from None + raise TimeoutError from None + + success, value = item + if success: + return value + raise value + + __next__ = next # XXX + + def _set(self, i, obj): + with self._cond: + if self._index == i: + self._items.append(obj) + self._index += 1 + while self._index in self._unsorted: + obj = self._unsorted.pop(self._index) + self._items.append(obj) + self._index += 1 + self._cond.notify() + else: + self._unsorted[i] = obj + + if self._index == self._length: + del self._cache[self._job] + self._pool = None + + def _set_length(self, length): + with self._cond: + self._length = length + if self._index == self._length: + self._cond.notify() + del self._cache[self._job] + self._pool = None + +# +# Class whose instances are returned by `Pool.imap_unordered()` +# + +class IMapUnorderedIterator(IMapIterator): + + def _set(self, i, obj): + with self._cond: + self._items.append(obj) + self._index += 1 + self._cond.notify() + if self._index == self._length: + del self._cache[self._job] + self._pool = None + +# +# +# + +class ThreadPool(Pool): + _wrap_exception = False + + @staticmethod + def Process(ctx, *args, **kwds): + from .dummy import Process + return Process(*args, **kwds) + + def __init__(self, processes=None, initializer=None, initargs=()): + Pool.__init__(self, processes, initializer, initargs) + + def _setup_queues(self): + self._inqueue = queue.SimpleQueue() + self._outqueue = queue.SimpleQueue() + self._quick_put = self._inqueue.put + self._quick_get = self._outqueue.get + + def _get_sentinels(self): + return [self._change_notifier._reader] + + @staticmethod + def _get_worker_sentinels(workers): + return [] + + @staticmethod + def _help_stuff_finish(inqueue, task_handler, size): + # drain inqueue, and put sentinels at its head to make workers finish + try: + while True: + inqueue.get(block=False) + except queue.Empty: + pass + for i in range(size): + inqueue.put(None) + + def _wait_for_updates(self, sentinels, change_notifier, timeout): + time.sleep(timeout) diff --git a/lib/python3.1/multiprocessing/popen_fork.py b/lib/python3.1/multiprocessing/popen_fork.py new file mode 100644 index 0000000000000000000000000000000000000000..625981cf47627cab9d03cae4d3c6d8224a9375cc --- /dev/null +++ b/lib/python3.1/multiprocessing/popen_fork.py @@ -0,0 +1,83 @@ +import os +import signal + +from . import util + +__all__ = ['Popen'] + +# +# Start child process using fork +# + +class Popen(object): + method = 'fork' + + def __init__(self, process_obj): + util._flush_std_streams() + self.returncode = None + self.finalizer = None + self._launch(process_obj) + + def duplicate_for_child(self, fd): + return fd + + def poll(self, flag=os.WNOHANG): + if self.returncode is None: + try: + pid, sts = os.waitpid(self.pid, flag) + except OSError: + # Child process not yet created. See #1731717 + # e.errno == errno.ECHILD == 10 + return None + if pid == self.pid: + self.returncode = os.waitstatus_to_exitcode(sts) + return self.returncode + + def wait(self, timeout=None): + if self.returncode is None: + if timeout is not None: + from multiprocessing.connection import wait + if not wait([self.sentinel], timeout): + return None + # This shouldn't block if wait() returned successfully. + return self.poll(os.WNOHANG if timeout == 0.0 else 0) + return self.returncode + + def _send_signal(self, sig): + if self.returncode is None: + try: + os.kill(self.pid, sig) + except ProcessLookupError: + pass + except OSError: + if self.wait(timeout=0.1) is None: + raise + + def terminate(self): + self._send_signal(signal.SIGTERM) + + def kill(self): + self._send_signal(signal.SIGKILL) + + def _launch(self, process_obj): + code = 1 + parent_r, child_w = os.pipe() + child_r, parent_w = os.pipe() + self.pid = os.fork() + if self.pid == 0: + try: + os.close(parent_r) + os.close(parent_w) + code = process_obj._bootstrap(parent_sentinel=child_r) + finally: + os._exit(code) + else: + os.close(child_w) + os.close(child_r) + self.finalizer = util.Finalize(self, util.close_fds, + (parent_r, parent_w,)) + self.sentinel = parent_r + + def close(self): + if self.finalizer is not None: + self.finalizer() diff --git a/lib/python3.1/multiprocessing/popen_forkserver.py b/lib/python3.1/multiprocessing/popen_forkserver.py new file mode 100644 index 0000000000000000000000000000000000000000..a56eb9bf11080b0d499ae81cfef7b501545cb90e --- /dev/null +++ b/lib/python3.1/multiprocessing/popen_forkserver.py @@ -0,0 +1,74 @@ +import io +import os + +from .context import reduction, set_spawning_popen +if not reduction.HAVE_SEND_HANDLE: + raise ImportError('No support for sending fds between processes') +from . import forkserver +from . import popen_fork +from . import spawn +from . import util + + +__all__ = ['Popen'] + +# +# Wrapper for an fd used while launching a process +# + +class _DupFd(object): + def __init__(self, ind): + self.ind = ind + def detach(self): + return forkserver.get_inherited_fds()[self.ind] + +# +# Start child process using a server process +# + +class Popen(popen_fork.Popen): + method = 'forkserver' + DupFd = _DupFd + + def __init__(self, process_obj): + self._fds = [] + super().__init__(process_obj) + + def duplicate_for_child(self, fd): + self._fds.append(fd) + return len(self._fds) - 1 + + def _launch(self, process_obj): + prep_data = spawn.get_preparation_data(process_obj._name) + buf = io.BytesIO() + set_spawning_popen(self) + try: + reduction.dump(prep_data, buf) + reduction.dump(process_obj, buf) + finally: + set_spawning_popen(None) + + self.sentinel, w = forkserver.connect_to_new_process(self._fds) + # Keep a duplicate of the data pipe's write end as a sentinel of the + # parent process used by the child process. + _parent_w = os.dup(w) + self.finalizer = util.Finalize(self, util.close_fds, + (_parent_w, self.sentinel)) + with open(w, 'wb', closefd=True) as f: + f.write(buf.getbuffer()) + self.pid = forkserver.read_signed(self.sentinel) + + def poll(self, flag=os.WNOHANG): + if self.returncode is None: + from multiprocessing.connection import wait + timeout = 0 if flag == os.WNOHANG else None + if not wait([self.sentinel], timeout): + return None + try: + self.returncode = forkserver.read_signed(self.sentinel) + except (OSError, EOFError): + # This should not happen usually, but perhaps the forkserver + # process itself got killed + self.returncode = 255 + + return self.returncode diff --git a/lib/python3.1/multiprocessing/popen_spawn_posix.py b/lib/python3.1/multiprocessing/popen_spawn_posix.py new file mode 100644 index 0000000000000000000000000000000000000000..24b8634523e5f2c29cd8bb21022c26d22a4fb13b --- /dev/null +++ b/lib/python3.1/multiprocessing/popen_spawn_posix.py @@ -0,0 +1,72 @@ +import io +import os + +from .context import reduction, set_spawning_popen +from . import popen_fork +from . import spawn +from . import util + +__all__ = ['Popen'] + + +# +# Wrapper for an fd used while launching a process +# + +class _DupFd(object): + def __init__(self, fd): + self.fd = fd + def detach(self): + return self.fd + +# +# Start child process using a fresh interpreter +# + +class Popen(popen_fork.Popen): + method = 'spawn' + DupFd = _DupFd + + def __init__(self, process_obj): + self._fds = [] + super().__init__(process_obj) + + def duplicate_for_child(self, fd): + self._fds.append(fd) + return fd + + def _launch(self, process_obj): + from . import resource_tracker + tracker_fd = resource_tracker.getfd() + self._fds.append(tracker_fd) + prep_data = spawn.get_preparation_data(process_obj._name) + fp = io.BytesIO() + set_spawning_popen(self) + try: + reduction.dump(prep_data, fp) + reduction.dump(process_obj, fp) + finally: + set_spawning_popen(None) + + parent_r = child_w = child_r = parent_w = None + try: + parent_r, child_w = os.pipe() + child_r, parent_w = os.pipe() + cmd = spawn.get_command_line(tracker_fd=tracker_fd, + pipe_handle=child_r) + self._fds.extend([child_r, child_w]) + self.pid = util.spawnv_passfds(spawn.get_executable(), + cmd, self._fds) + self.sentinel = parent_r + with open(parent_w, 'wb', closefd=False) as f: + f.write(fp.getbuffer()) + finally: + fds_to_close = [] + for fd in (parent_r, parent_w): + if fd is not None: + fds_to_close.append(fd) + self.finalizer = util.Finalize(self, util.close_fds, fds_to_close) + + for fd in (child_r, child_w): + if fd is not None: + os.close(fd) diff --git a/lib/python3.1/multiprocessing/popen_spawn_win32.py b/lib/python3.1/multiprocessing/popen_spawn_win32.py new file mode 100644 index 0000000000000000000000000000000000000000..9c4098d0fa4f1e6e3ec94ecc8e596dd3857d741f --- /dev/null +++ b/lib/python3.1/multiprocessing/popen_spawn_win32.py @@ -0,0 +1,131 @@ +import os +import msvcrt +import signal +import sys +import _winapi + +from .context import reduction, get_spawning_popen, set_spawning_popen +from . import spawn +from . import util + +__all__ = ['Popen'] + +# +# +# + +TERMINATE = 0x10000 +WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) +WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") + + +def _path_eq(p1, p2): + return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2) + +WINENV = not _path_eq(sys.executable, sys._base_executable) + + +def _close_handles(*handles): + for handle in handles: + _winapi.CloseHandle(handle) + + +# +# We define a Popen class similar to the one from subprocess, but +# whose constructor takes a process object as its argument. +# + +class Popen(object): + ''' + Start a subprocess to run the code of a process object + ''' + method = 'spawn' + + def __init__(self, process_obj): + prep_data = spawn.get_preparation_data(process_obj._name) + + # read end of pipe will be duplicated by the child process + # -- see spawn_main() in spawn.py. + # + # bpo-33929: Previously, the read end of pipe was "stolen" by the child + # process, but it leaked a handle if the child process had been + # terminated before it could steal the handle from the parent process. + rhandle, whandle = _winapi.CreatePipe(None, 0) + wfd = msvcrt.open_osfhandle(whandle, 0) + cmd = spawn.get_command_line(parent_pid=os.getpid(), + pipe_handle=rhandle) + cmd = ' '.join('"%s"' % x for x in cmd) + + python_exe = spawn.get_executable() + + # bpo-35797: When running in a venv, we bypass the redirect + # executor and launch our base Python. + if WINENV and _path_eq(python_exe, sys.executable): + python_exe = sys._base_executable + env = os.environ.copy() + env["__PYVENV_LAUNCHER__"] = sys.executable + else: + env = None + + with open(wfd, 'wb', closefd=True) as to_child: + # start process + try: + hp, ht, pid, tid = _winapi.CreateProcess( + python_exe, cmd, + None, None, False, 0, env, None, None) + _winapi.CloseHandle(ht) + except: + _winapi.CloseHandle(rhandle) + raise + + # set attributes of self + self.pid = pid + self.returncode = None + self._handle = hp + self.sentinel = int(hp) + self.finalizer = util.Finalize(self, _close_handles, + (self.sentinel, int(rhandle))) + + # send information to child + set_spawning_popen(self) + try: + reduction.dump(prep_data, to_child) + reduction.dump(process_obj, to_child) + finally: + set_spawning_popen(None) + + def duplicate_for_child(self, handle): + assert self is get_spawning_popen() + return reduction.duplicate(handle, self.sentinel) + + def wait(self, timeout=None): + if self.returncode is None: + if timeout is None: + msecs = _winapi.INFINITE + else: + msecs = max(0, int(timeout * 1000 + 0.5)) + + res = _winapi.WaitForSingleObject(int(self._handle), msecs) + if res == _winapi.WAIT_OBJECT_0: + code = _winapi.GetExitCodeProcess(self._handle) + if code == TERMINATE: + code = -signal.SIGTERM + self.returncode = code + + return self.returncode + + def poll(self): + return self.wait(timeout=0) + + def terminate(self): + if self.returncode is None: + try: + _winapi.TerminateProcess(int(self._handle), TERMINATE) + except OSError: + if self.wait(timeout=1.0) is None: + raise + + kill = terminate + + def close(self): + self.finalizer() diff --git a/lib/python3.1/multiprocessing/process.py b/lib/python3.1/multiprocessing/process.py new file mode 100644 index 0000000000000000000000000000000000000000..2d5372e6d947c849eedef02b7885acfd25930e12 --- /dev/null +++ b/lib/python3.1/multiprocessing/process.py @@ -0,0 +1,438 @@ +# +# Module providing the `Process` class which emulates `threading.Thread` +# +# multiprocessing/process.py +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +__all__ = ['BaseProcess', 'current_process', 'active_children', + 'parent_process'] + +# +# Imports +# + +import os +import sys +import signal +import itertools +import threading +from _weakrefset import WeakSet + +# +# +# + +try: + ORIGINAL_DIR = os.path.abspath(os.getcwd()) +except OSError: + ORIGINAL_DIR = None + +# +# Public functions +# + +def current_process(): + ''' + Return process object representing the current process + ''' + return _current_process + +def active_children(): + ''' + Return list of process objects corresponding to live child processes + ''' + _cleanup() + return list(_children) + + +def parent_process(): + ''' + Return process object representing the parent process + ''' + return _parent_process + +# +# +# + +def _cleanup(): + # check for processes which have finished + for p in list(_children): + if p._popen.poll() is not None: + _children.discard(p) + +# +# The `Process` class +# + +class BaseProcess(object): + ''' + Process objects represent activity that is run in a separate process + + The class is analogous to `threading.Thread` + ''' + def _Popen(self): + raise NotImplementedError + + def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, + *, daemon=None): + assert group is None, 'group argument must be None for now' + count = next(_process_counter) + self._identity = _current_process._identity + (count,) + self._config = _current_process._config.copy() + self._parent_pid = os.getpid() + self._parent_name = _current_process.name + self._popen = None + self._closed = False + self._target = target + self._args = tuple(args) + self._kwargs = dict(kwargs) + self._name = name or type(self).__name__ + '-' + \ + ':'.join(str(i) for i in self._identity) + if daemon is not None: + self.daemon = daemon + _dangling.add(self) + + def _check_closed(self): + if self._closed: + raise ValueError("process object is closed") + + def run(self): + ''' + Method to be run in sub-process; can be overridden in sub-class + ''' + if self._target: + self._target(*self._args, **self._kwargs) + + def start(self): + ''' + Start child process + ''' + self._check_closed() + assert self._popen is None, 'cannot start a process twice' + assert self._parent_pid == os.getpid(), \ + 'can only start a process object created by current process' + assert not _current_process._config.get('daemon'), \ + 'daemonic processes are not allowed to have children' + _cleanup() + self._popen = self._Popen(self) + self._sentinel = self._popen.sentinel + # Avoid a refcycle if the target function holds an indirect + # reference to the process object (see bpo-30775) + del self._target, self._args, self._kwargs + _children.add(self) + + def terminate(self): + ''' + Terminate process; sends SIGTERM signal or uses TerminateProcess() + ''' + self._check_closed() + self._popen.terminate() + + def kill(self): + ''' + Terminate process; sends SIGKILL signal or uses TerminateProcess() + ''' + self._check_closed() + self._popen.kill() + + def join(self, timeout=None): + ''' + Wait until child process terminates + ''' + self._check_closed() + assert self._parent_pid == os.getpid(), 'can only join a child process' + assert self._popen is not None, 'can only join a started process' + res = self._popen.wait(timeout) + if res is not None: + _children.discard(self) + + def is_alive(self): + ''' + Return whether process is alive + ''' + self._check_closed() + if self is _current_process: + return True + assert self._parent_pid == os.getpid(), 'can only test a child process' + + if self._popen is None: + return False + + returncode = self._popen.poll() + if returncode is None: + return True + else: + _children.discard(self) + return False + + def close(self): + ''' + Close the Process object. + + This method releases resources held by the Process object. It is + an error to call this method if the child process is still running. + ''' + if self._popen is not None: + if self._popen.poll() is None: + raise ValueError("Cannot close a process while it is still running. " + "You should first call join() or terminate().") + self._popen.close() + self._popen = None + del self._sentinel + _children.discard(self) + self._closed = True + + @property + def name(self): + return self._name + + @name.setter + def name(self, name): + assert isinstance(name, str), 'name must be a string' + self._name = name + + @property + def daemon(self): + ''' + Return whether process is a daemon + ''' + return self._config.get('daemon', False) + + @daemon.setter + def daemon(self, daemonic): + ''' + Set whether process is a daemon + ''' + assert self._popen is None, 'process has already started' + self._config['daemon'] = daemonic + + @property + def authkey(self): + return self._config['authkey'] + + @authkey.setter + def authkey(self, authkey): + ''' + Set authorization key of process + ''' + self._config['authkey'] = AuthenticationString(authkey) + + @property + def exitcode(self): + ''' + Return exit code of process or `None` if it has yet to stop + ''' + self._check_closed() + if self._popen is None: + return self._popen + return self._popen.poll() + + @property + def ident(self): + ''' + Return identifier (PID) of process or `None` if it has yet to start + ''' + self._check_closed() + if self is _current_process: + return os.getpid() + else: + return self._popen and self._popen.pid + + pid = ident + + @property + def sentinel(self): + ''' + Return a file descriptor (Unix) or handle (Windows) suitable for + waiting for process termination. + ''' + self._check_closed() + try: + return self._sentinel + except AttributeError: + raise ValueError("process not started") from None + + def __repr__(self): + exitcode = None + if self is _current_process: + status = 'started' + elif self._closed: + status = 'closed' + elif self._parent_pid != os.getpid(): + status = 'unknown' + elif self._popen is None: + status = 'initial' + else: + exitcode = self._popen.poll() + if exitcode is not None: + status = 'stopped' + else: + status = 'started' + + info = [type(self).__name__, 'name=%r' % self._name] + if self._popen is not None: + info.append('pid=%s' % self._popen.pid) + info.append('parent=%s' % self._parent_pid) + info.append(status) + if exitcode is not None: + exitcode = _exitcode_to_name.get(exitcode, exitcode) + info.append('exitcode=%s' % exitcode) + if self.daemon: + info.append('daemon') + return '<%s>' % ' '.join(info) + + ## + + def _bootstrap(self, parent_sentinel=None): + from . import util, context + global _current_process, _parent_process, _process_counter, _children + + try: + if self._start_method is not None: + context._force_start_method(self._start_method) + _process_counter = itertools.count(1) + _children = set() + util._close_stdin() + old_process = _current_process + _current_process = self + _parent_process = _ParentProcess( + self._parent_name, self._parent_pid, parent_sentinel) + if threading._HAVE_THREAD_NATIVE_ID: + threading.main_thread()._set_native_id() + try: + self._after_fork() + finally: + # delay finalization of the old process object until after + # _run_after_forkers() is executed + del old_process + util.info('child process calling self.run()') + try: + self.run() + exitcode = 0 + finally: + util._exit_function() + except SystemExit as e: + if e.code is None: + exitcode = 0 + elif isinstance(e.code, int): + exitcode = e.code + else: + sys.stderr.write(str(e.code) + '\n') + exitcode = 1 + except: + exitcode = 1 + import traceback + sys.stderr.write('Process %s:\n' % self.name) + traceback.print_exc() + finally: + threading._shutdown() + util.info('process exiting with exitcode %d' % exitcode) + util._flush_std_streams() + + return exitcode + + @staticmethod + def _after_fork(): + from . import util + util._finalizer_registry.clear() + util._run_after_forkers() + + +# +# We subclass bytes to avoid accidental transmission of auth keys over network +# + +class AuthenticationString(bytes): + def __reduce__(self): + from .context import get_spawning_popen + if get_spawning_popen() is None: + raise TypeError( + 'Pickling an AuthenticationString object is ' + 'disallowed for security reasons' + ) + return AuthenticationString, (bytes(self),) + + +# +# Create object representing the parent process +# + +class _ParentProcess(BaseProcess): + + def __init__(self, name, pid, sentinel): + self._identity = () + self._name = name + self._pid = pid + self._parent_pid = None + self._popen = None + self._closed = False + self._sentinel = sentinel + self._config = {} + + def is_alive(self): + from multiprocessing.connection import wait + return not wait([self._sentinel], timeout=0) + + @property + def ident(self): + return self._pid + + def join(self, timeout=None): + ''' + Wait until parent process terminates + ''' + from multiprocessing.connection import wait + wait([self._sentinel], timeout=timeout) + + pid = ident + +# +# Create object representing the main process +# + +class _MainProcess(BaseProcess): + + def __init__(self): + self._identity = () + self._name = 'MainProcess' + self._parent_pid = None + self._popen = None + self._closed = False + self._config = {'authkey': AuthenticationString(os.urandom(32)), + 'semprefix': '/mp'} + # Note that some versions of FreeBSD only allow named + # semaphores to have names of up to 14 characters. Therefore + # we choose a short prefix. + # + # On MacOSX in a sandbox it may be necessary to use a + # different prefix -- see #19478. + # + # Everything in self._config will be inherited by descendant + # processes. + + def close(self): + pass + + +_parent_process = None +_current_process = _MainProcess() +_process_counter = itertools.count(1) +_children = set() +del _MainProcess + +# +# Give names to some return codes +# + +_exitcode_to_name = {} + +for name, signum in list(signal.__dict__.items()): + if name[:3]=='SIG' and '_' not in name: + _exitcode_to_name[-signum] = f'-{name}' + +# For debug and leak testing +_dangling = WeakSet() diff --git a/lib/python3.1/multiprocessing/queues.py b/lib/python3.1/multiprocessing/queues.py new file mode 100644 index 0000000000000000000000000000000000000000..f37f114a968871ba367ac33d5f0b109da8869834 --- /dev/null +++ b/lib/python3.1/multiprocessing/queues.py @@ -0,0 +1,379 @@ +# +# Module implementing queues +# +# multiprocessing/queues.py +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +__all__ = ['Queue', 'SimpleQueue', 'JoinableQueue'] + +import sys +import os +import threading +import collections +import time +import types +import weakref +import errno + +from queue import Empty, Full + +import _multiprocessing + +from . import connection +from . import context +_ForkingPickler = context.reduction.ForkingPickler + +from .util import debug, info, Finalize, register_after_fork, is_exiting + +# +# Queue type using a pipe, buffer and thread +# + +class Queue(object): + + def __init__(self, maxsize=0, *, ctx): + if maxsize <= 0: + # Can raise ImportError (see issues #3770 and #23400) + from .synchronize import SEM_VALUE_MAX as maxsize + self._maxsize = maxsize + self._reader, self._writer = connection.Pipe(duplex=False) + self._rlock = ctx.Lock() + self._opid = os.getpid() + if sys.platform == 'win32': + self._wlock = None + else: + self._wlock = ctx.Lock() + self._sem = ctx.BoundedSemaphore(maxsize) + # For use by concurrent.futures + self._ignore_epipe = False + self._reset() + + if sys.platform != 'win32': + register_after_fork(self, Queue._after_fork) + + def __getstate__(self): + context.assert_spawning(self) + return (self._ignore_epipe, self._maxsize, self._reader, self._writer, + self._rlock, self._wlock, self._sem, self._opid) + + def __setstate__(self, state): + (self._ignore_epipe, self._maxsize, self._reader, self._writer, + self._rlock, self._wlock, self._sem, self._opid) = state + self._reset() + + def _after_fork(self): + debug('Queue._after_fork()') + self._reset(after_fork=True) + + def _reset(self, after_fork=False): + if after_fork: + self._notempty._at_fork_reinit() + else: + self._notempty = threading.Condition(threading.Lock()) + self._buffer = collections.deque() + self._thread = None + self._jointhread = None + self._joincancelled = False + self._closed = False + self._close = None + self._send_bytes = self._writer.send_bytes + self._recv_bytes = self._reader.recv_bytes + self._poll = self._reader.poll + + def put(self, obj, block=True, timeout=None): + if self._closed: + raise ValueError(f"Queue {self!r} is closed") + if not self._sem.acquire(block, timeout): + raise Full + + with self._notempty: + if self._thread is None: + self._start_thread() + self._buffer.append(obj) + self._notempty.notify() + + def get(self, block=True, timeout=None): + if self._closed: + raise ValueError(f"Queue {self!r} is closed") + if block and timeout is None: + with self._rlock: + res = self._recv_bytes() + self._sem.release() + else: + if block: + deadline = time.monotonic() + timeout + if not self._rlock.acquire(block, timeout): + raise Empty + try: + if block: + timeout = deadline - time.monotonic() + if not self._poll(timeout): + raise Empty + elif not self._poll(): + raise Empty + res = self._recv_bytes() + self._sem.release() + finally: + self._rlock.release() + # unserialize the data after having released the lock + return _ForkingPickler.loads(res) + + def qsize(self): + # Raises NotImplementedError on Mac OSX because of broken sem_getvalue() + return self._maxsize - self._sem._semlock._get_value() + + def empty(self): + return not self._poll() + + def full(self): + return self._sem._semlock._is_zero() + + def get_nowait(self): + return self.get(False) + + def put_nowait(self, obj): + return self.put(obj, False) + + def close(self): + self._closed = True + close = self._close + if close: + self._close = None + close() + + def join_thread(self): + debug('Queue.join_thread()') + assert self._closed, "Queue {0!r} not closed".format(self) + if self._jointhread: + self._jointhread() + + def cancel_join_thread(self): + debug('Queue.cancel_join_thread()') + self._joincancelled = True + try: + self._jointhread.cancel() + except AttributeError: + pass + + def _start_thread(self): + debug('Queue._start_thread()') + + # Start thread which transfers data from buffer to pipe + self._buffer.clear() + self._thread = threading.Thread( + target=Queue._feed, + args=(self._buffer, self._notempty, self._send_bytes, + self._wlock, self._reader.close, self._writer.close, + self._ignore_epipe, self._on_queue_feeder_error, + self._sem), + name='QueueFeederThread' + ) + self._thread.daemon = True + + debug('doing self._thread.start()') + self._thread.start() + debug('... done self._thread.start()') + + if not self._joincancelled: + self._jointhread = Finalize( + self._thread, Queue._finalize_join, + [weakref.ref(self._thread)], + exitpriority=-5 + ) + + # Send sentinel to the thread queue object when garbage collected + self._close = Finalize( + self, Queue._finalize_close, + [self._buffer, self._notempty], + exitpriority=10 + ) + + @staticmethod + def _finalize_join(twr): + debug('joining queue thread') + thread = twr() + if thread is not None: + thread.join() + debug('... queue thread joined') + else: + debug('... queue thread already dead') + + @staticmethod + def _finalize_close(buffer, notempty): + debug('telling queue thread to quit') + with notempty: + buffer.append(_sentinel) + notempty.notify() + + @staticmethod + def _feed(buffer, notempty, send_bytes, writelock, reader_close, + writer_close, ignore_epipe, onerror, queue_sem): + debug('starting thread to feed data to pipe') + nacquire = notempty.acquire + nrelease = notempty.release + nwait = notempty.wait + bpopleft = buffer.popleft + sentinel = _sentinel + if sys.platform != 'win32': + wacquire = writelock.acquire + wrelease = writelock.release + else: + wacquire = None + + while 1: + try: + nacquire() + try: + if not buffer: + nwait() + finally: + nrelease() + try: + while 1: + obj = bpopleft() + if obj is sentinel: + debug('feeder thread got sentinel -- exiting') + reader_close() + writer_close() + return + + # serialize the data before acquiring the lock + obj = _ForkingPickler.dumps(obj) + if wacquire is None: + send_bytes(obj) + else: + wacquire() + try: + send_bytes(obj) + finally: + wrelease() + except IndexError: + pass + except Exception as e: + if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: + return + # Since this runs in a daemon thread the resources it uses + # may be become unusable while the process is cleaning up. + # We ignore errors which happen after the process has + # started to cleanup. + if is_exiting(): + info('error in queue thread: %s', e) + return + else: + # Since the object has not been sent in the queue, we need + # to decrease the size of the queue. The error acts as + # if the object had been silently removed from the queue + # and this step is necessary to have a properly working + # queue. + queue_sem.release() + onerror(e, obj) + + @staticmethod + def _on_queue_feeder_error(e, obj): + """ + Private API hook called when feeding data in the background thread + raises an exception. For overriding by concurrent.futures. + """ + import traceback + traceback.print_exc() + + +_sentinel = object() + +# +# A queue type which also supports join() and task_done() methods +# +# Note that if you do not call task_done() for each finished task then +# eventually the counter's semaphore may overflow causing Bad Things +# to happen. +# + +class JoinableQueue(Queue): + + def __init__(self, maxsize=0, *, ctx): + Queue.__init__(self, maxsize, ctx=ctx) + self._unfinished_tasks = ctx.Semaphore(0) + self._cond = ctx.Condition() + + def __getstate__(self): + return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks) + + def __setstate__(self, state): + Queue.__setstate__(self, state[:-2]) + self._cond, self._unfinished_tasks = state[-2:] + + def put(self, obj, block=True, timeout=None): + if self._closed: + raise ValueError(f"Queue {self!r} is closed") + if not self._sem.acquire(block, timeout): + raise Full + + with self._notempty, self._cond: + if self._thread is None: + self._start_thread() + self._buffer.append(obj) + self._unfinished_tasks.release() + self._notempty.notify() + + def task_done(self): + with self._cond: + if not self._unfinished_tasks.acquire(False): + raise ValueError('task_done() called too many times') + if self._unfinished_tasks._semlock._is_zero(): + self._cond.notify_all() + + def join(self): + with self._cond: + if not self._unfinished_tasks._semlock._is_zero(): + self._cond.wait() + +# +# Simplified Queue type -- really just a locked pipe +# + +class SimpleQueue(object): + + def __init__(self, *, ctx): + self._reader, self._writer = connection.Pipe(duplex=False) + self._rlock = ctx.Lock() + self._poll = self._reader.poll + if sys.platform == 'win32': + self._wlock = None + else: + self._wlock = ctx.Lock() + + def close(self): + self._reader.close() + self._writer.close() + + def empty(self): + return not self._poll() + + def __getstate__(self): + context.assert_spawning(self) + return (self._reader, self._writer, self._rlock, self._wlock) + + def __setstate__(self, state): + (self._reader, self._writer, self._rlock, self._wlock) = state + self._poll = self._reader.poll + + def get(self): + with self._rlock: + res = self._reader.recv_bytes() + # unserialize the data after having released the lock + return _ForkingPickler.loads(res) + + def put(self, obj): + # serialize the data before acquiring the lock + obj = _ForkingPickler.dumps(obj) + if self._wlock is None: + # writes to a message oriented win32 pipe are atomic + self._writer.send_bytes(obj) + else: + with self._wlock: + self._writer.send_bytes(obj) + + __class_getitem__ = classmethod(types.GenericAlias) diff --git a/lib/python3.1/multiprocessing/reduction.py b/lib/python3.1/multiprocessing/reduction.py new file mode 100644 index 0000000000000000000000000000000000000000..5593f0682f7fce3dc90402187c1fd404aa2f8a42 --- /dev/null +++ b/lib/python3.1/multiprocessing/reduction.py @@ -0,0 +1,281 @@ +# +# Module which deals with pickling of objects. +# +# multiprocessing/reduction.py +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +from abc import ABCMeta +import copyreg +import functools +import io +import os +import pickle +import socket +import sys + +from . import context + +__all__ = ['send_handle', 'recv_handle', 'ForkingPickler', 'register', 'dump'] + + +HAVE_SEND_HANDLE = (sys.platform == 'win32' or + (hasattr(socket, 'CMSG_LEN') and + hasattr(socket, 'SCM_RIGHTS') and + hasattr(socket.socket, 'sendmsg'))) + +# +# Pickler subclass +# + +class ForkingPickler(pickle.Pickler): + '''Pickler subclass used by multiprocessing.''' + _extra_reducers = {} + _copyreg_dispatch_table = copyreg.dispatch_table + + def __init__(self, *args): + super().__init__(*args) + self.dispatch_table = self._copyreg_dispatch_table.copy() + self.dispatch_table.update(self._extra_reducers) + + @classmethod + def register(cls, type, reduce): + '''Register a reduce function for a type.''' + cls._extra_reducers[type] = reduce + + @classmethod + def dumps(cls, obj, protocol=None): + buf = io.BytesIO() + cls(buf, protocol).dump(obj) + return buf.getbuffer() + + loads = pickle.loads + +register = ForkingPickler.register + +def dump(obj, file, protocol=None): + '''Replacement for pickle.dump() using ForkingPickler.''' + ForkingPickler(file, protocol).dump(obj) + +# +# Platform specific definitions +# + +if sys.platform == 'win32': + # Windows + __all__ += ['DupHandle', 'duplicate', 'steal_handle'] + import _winapi + + def duplicate(handle, target_process=None, inheritable=False, + *, source_process=None): + '''Duplicate a handle. (target_process is a handle not a pid!)''' + current_process = _winapi.GetCurrentProcess() + if source_process is None: + source_process = current_process + if target_process is None: + target_process = current_process + return _winapi.DuplicateHandle( + source_process, handle, target_process, + 0, inheritable, _winapi.DUPLICATE_SAME_ACCESS) + + def steal_handle(source_pid, handle): + '''Steal a handle from process identified by source_pid.''' + source_process_handle = _winapi.OpenProcess( + _winapi.PROCESS_DUP_HANDLE, False, source_pid) + try: + return _winapi.DuplicateHandle( + source_process_handle, handle, + _winapi.GetCurrentProcess(), 0, False, + _winapi.DUPLICATE_SAME_ACCESS | _winapi.DUPLICATE_CLOSE_SOURCE) + finally: + _winapi.CloseHandle(source_process_handle) + + def send_handle(conn, handle, destination_pid): + '''Send a handle over a local connection.''' + dh = DupHandle(handle, _winapi.DUPLICATE_SAME_ACCESS, destination_pid) + conn.send(dh) + + def recv_handle(conn): + '''Receive a handle over a local connection.''' + return conn.recv().detach() + + class DupHandle(object): + '''Picklable wrapper for a handle.''' + def __init__(self, handle, access, pid=None): + if pid is None: + # We just duplicate the handle in the current process and + # let the receiving process steal the handle. + pid = os.getpid() + proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, pid) + try: + self._handle = _winapi.DuplicateHandle( + _winapi.GetCurrentProcess(), + handle, proc, access, False, 0) + finally: + _winapi.CloseHandle(proc) + self._access = access + self._pid = pid + + def detach(self): + '''Get the handle. This should only be called once.''' + # retrieve handle from process which currently owns it + if self._pid == os.getpid(): + # The handle has already been duplicated for this process. + return self._handle + # We must steal the handle from the process whose pid is self._pid. + proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, + self._pid) + try: + return _winapi.DuplicateHandle( + proc, self._handle, _winapi.GetCurrentProcess(), + self._access, False, _winapi.DUPLICATE_CLOSE_SOURCE) + finally: + _winapi.CloseHandle(proc) + +else: + # Unix + __all__ += ['DupFd', 'sendfds', 'recvfds'] + import array + + # On MacOSX we should acknowledge receipt of fds -- see Issue14669 + ACKNOWLEDGE = sys.platform == 'darwin' + + def sendfds(sock, fds): + '''Send an array of fds over an AF_UNIX socket.''' + fds = array.array('i', fds) + msg = bytes([len(fds) % 256]) + sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)]) + if ACKNOWLEDGE and sock.recv(1) != b'A': + raise RuntimeError('did not receive acknowledgement of fd') + + def recvfds(sock, size): + '''Receive an array of fds over an AF_UNIX socket.''' + a = array.array('i') + bytes_size = a.itemsize * size + msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_SPACE(bytes_size)) + if not msg and not ancdata: + raise EOFError + try: + if ACKNOWLEDGE: + sock.send(b'A') + if len(ancdata) != 1: + raise RuntimeError('received %d items of ancdata' % + len(ancdata)) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + if (cmsg_level == socket.SOL_SOCKET and + cmsg_type == socket.SCM_RIGHTS): + if len(cmsg_data) % a.itemsize != 0: + raise ValueError + a.frombytes(cmsg_data) + if len(a) % 256 != msg[0]: + raise AssertionError( + "Len is {0:n} but msg[0] is {1!r}".format( + len(a), msg[0])) + return list(a) + except (ValueError, IndexError): + pass + raise RuntimeError('Invalid data received') + + def send_handle(conn, handle, destination_pid): + '''Send a handle over a local connection.''' + with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s: + sendfds(s, [handle]) + + def recv_handle(conn): + '''Receive a handle over a local connection.''' + with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s: + return recvfds(s, 1)[0] + + def DupFd(fd): + '''Return a wrapper for an fd.''' + popen_obj = context.get_spawning_popen() + if popen_obj is not None: + return popen_obj.DupFd(popen_obj.duplicate_for_child(fd)) + elif HAVE_SEND_HANDLE: + from . import resource_sharer + return resource_sharer.DupFd(fd) + else: + raise ValueError('SCM_RIGHTS appears not to be available') + +# +# Try making some callable types picklable +# + +def _reduce_method(m): + if m.__self__ is None: + return getattr, (m.__class__, m.__func__.__name__) + else: + return getattr, (m.__self__, m.__func__.__name__) +class _C: + def f(self): + pass +register(type(_C().f), _reduce_method) + + +def _reduce_method_descriptor(m): + return getattr, (m.__objclass__, m.__name__) +register(type(list.append), _reduce_method_descriptor) +register(type(int.__add__), _reduce_method_descriptor) + + +def _reduce_partial(p): + return _rebuild_partial, (p.func, p.args, p.keywords or {}) +def _rebuild_partial(func, args, keywords): + return functools.partial(func, *args, **keywords) +register(functools.partial, _reduce_partial) + +# +# Make sockets picklable +# + +if sys.platform == 'win32': + def _reduce_socket(s): + from .resource_sharer import DupSocket + return _rebuild_socket, (DupSocket(s),) + def _rebuild_socket(ds): + return ds.detach() + register(socket.socket, _reduce_socket) + +else: + def _reduce_socket(s): + df = DupFd(s.fileno()) + return _rebuild_socket, (df, s.family, s.type, s.proto) + def _rebuild_socket(df, family, type, proto): + fd = df.detach() + return socket.socket(family, type, proto, fileno=fd) + register(socket.socket, _reduce_socket) + + +class AbstractReducer(metaclass=ABCMeta): + '''Abstract base class for use in implementing a Reduction class + suitable for use in replacing the standard reduction mechanism + used in multiprocessing.''' + ForkingPickler = ForkingPickler + register = register + dump = dump + send_handle = send_handle + recv_handle = recv_handle + + if sys.platform == 'win32': + steal_handle = steal_handle + duplicate = duplicate + DupHandle = DupHandle + else: + sendfds = sendfds + recvfds = recvfds + DupFd = DupFd + + _reduce_method = _reduce_method + _reduce_method_descriptor = _reduce_method_descriptor + _rebuild_partial = _rebuild_partial + _reduce_socket = _reduce_socket + _rebuild_socket = _rebuild_socket + + def __init__(self, *args): + register(type(_C().f), _reduce_method) + register(type(list.append), _reduce_method_descriptor) + register(type(int.__add__), _reduce_method_descriptor) + register(functools.partial, _reduce_partial) + register(socket.socket, _reduce_socket) diff --git a/lib/python3.1/multiprocessing/resource_sharer.py b/lib/python3.1/multiprocessing/resource_sharer.py new file mode 100644 index 0000000000000000000000000000000000000000..66076509a1202e7a1b4d8a481f64621a4bfbbf3e --- /dev/null +++ b/lib/python3.1/multiprocessing/resource_sharer.py @@ -0,0 +1,154 @@ +# +# We use a background thread for sharing fds on Unix, and for sharing sockets on +# Windows. +# +# A client which wants to pickle a resource registers it with the resource +# sharer and gets an identifier in return. The unpickling process will connect +# to the resource sharer, sends the identifier and its pid, and then receives +# the resource. +# + +import os +import signal +import socket +import sys +import threading + +from . import process +from .context import reduction +from . import util + +__all__ = ['stop'] + + +if sys.platform == 'win32': + __all__ += ['DupSocket'] + + class DupSocket(object): + '''Picklable wrapper for a socket.''' + def __init__(self, sock): + new_sock = sock.dup() + def send(conn, pid): + share = new_sock.share(pid) + conn.send_bytes(share) + self._id = _resource_sharer.register(send, new_sock.close) + + def detach(self): + '''Get the socket. This should only be called once.''' + with _resource_sharer.get_connection(self._id) as conn: + share = conn.recv_bytes() + return socket.fromshare(share) + +else: + __all__ += ['DupFd'] + + class DupFd(object): + '''Wrapper for fd which can be used at any time.''' + def __init__(self, fd): + new_fd = os.dup(fd) + def send(conn, pid): + reduction.send_handle(conn, new_fd, pid) + def close(): + os.close(new_fd) + self._id = _resource_sharer.register(send, close) + + def detach(self): + '''Get the fd. This should only be called once.''' + with _resource_sharer.get_connection(self._id) as conn: + return reduction.recv_handle(conn) + + +class _ResourceSharer(object): + '''Manager for resources using background thread.''' + def __init__(self): + self._key = 0 + self._cache = {} + self._lock = threading.Lock() + self._listener = None + self._address = None + self._thread = None + util.register_after_fork(self, _ResourceSharer._afterfork) + + def register(self, send, close): + '''Register resource, returning an identifier.''' + with self._lock: + if self._address is None: + self._start() + self._key += 1 + self._cache[self._key] = (send, close) + return (self._address, self._key) + + @staticmethod + def get_connection(ident): + '''Return connection from which to receive identified resource.''' + from .connection import Client + address, key = ident + c = Client(address, authkey=process.current_process().authkey) + c.send((key, os.getpid())) + return c + + def stop(self, timeout=None): + '''Stop the background thread and clear registered resources.''' + from .connection import Client + with self._lock: + if self._address is not None: + c = Client(self._address, + authkey=process.current_process().authkey) + c.send(None) + c.close() + self._thread.join(timeout) + if self._thread.is_alive(): + util.sub_warning('_ResourceSharer thread did ' + 'not stop when asked') + self._listener.close() + self._thread = None + self._address = None + self._listener = None + for key, (send, close) in self._cache.items(): + close() + self._cache.clear() + + def _afterfork(self): + for key, (send, close) in self._cache.items(): + close() + self._cache.clear() + self._lock._at_fork_reinit() + if self._listener is not None: + self._listener.close() + self._listener = None + self._address = None + self._thread = None + + def _start(self): + from .connection import Listener + assert self._listener is None, "Already have Listener" + util.debug('starting listener and thread for sending handles') + self._listener = Listener(authkey=process.current_process().authkey) + self._address = self._listener.address + t = threading.Thread(target=self._serve) + t.daemon = True + t.start() + self._thread = t + + def _serve(self): + if hasattr(signal, 'pthread_sigmask'): + signal.pthread_sigmask(signal.SIG_BLOCK, signal.valid_signals()) + while 1: + try: + with self._listener.accept() as conn: + msg = conn.recv() + if msg is None: + break + key, destination_pid = msg + send, close = self._cache.pop(key) + try: + send(conn, destination_pid) + finally: + close() + except: + if not util.is_exiting(): + sys.excepthook(*sys.exc_info()) + + +_resource_sharer = _ResourceSharer() +stop = _resource_sharer.stop diff --git a/lib/python3.1/multiprocessing/resource_tracker.py b/lib/python3.1/multiprocessing/resource_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..ea369507297f86ca97ff289712da340a3b51fdf3 --- /dev/null +++ b/lib/python3.1/multiprocessing/resource_tracker.py @@ -0,0 +1,239 @@ +############################################################################### +# Server process to keep track of unlinked resources (like shared memory +# segments, semaphores etc.) and clean them. +# +# On Unix we run a server process which keeps track of unlinked +# resources. The server ignores SIGINT and SIGTERM and reads from a +# pipe. Every other process of the program has a copy of the writable +# end of the pipe, so we get EOF when all other processes have exited. +# Then the server process unlinks any remaining resource names. +# +# This is important because there may be system limits for such resources: for +# instance, the system only supports a limited number of named semaphores, and +# shared-memory segments live in the RAM. If a python process leaks such a +# resource, this resource will not be removed till the next reboot. Without +# this resource tracker process, "killall python" would probably leave unlinked +# resources. + +import os +import signal +import sys +import threading +import warnings + +from . import spawn +from . import util + +__all__ = ['ensure_running', 'register', 'unregister'] + +_HAVE_SIGMASK = hasattr(signal, 'pthread_sigmask') +_IGNORED_SIGNALS = (signal.SIGINT, signal.SIGTERM) + +_CLEANUP_FUNCS = { + 'noop': lambda: None, +} + +if os.name == 'posix': + import _multiprocessing + import _posixshmem + + # Use sem_unlink() to clean up named semaphores. + # + # sem_unlink() may be missing if the Python build process detected the + # absence of POSIX named semaphores. In that case, no named semaphores were + # ever opened, so no cleanup would be necessary. + if hasattr(_multiprocessing, 'sem_unlink'): + _CLEANUP_FUNCS.update({ + 'semaphore': _multiprocessing.sem_unlink, + }) + _CLEANUP_FUNCS.update({ + 'shared_memory': _posixshmem.shm_unlink, + }) + + +class ResourceTracker(object): + + def __init__(self): + self._lock = threading.Lock() + self._fd = None + self._pid = None + + def _stop(self): + with self._lock: + if self._fd is None: + # not running + return + + # closing the "alive" file descriptor stops main() + os.close(self._fd) + self._fd = None + + os.waitpid(self._pid, 0) + self._pid = None + + def getfd(self): + self.ensure_running() + return self._fd + + def ensure_running(self): + '''Make sure that resource tracker process is running. + + This can be run from any process. Usually a child process will use + the resource created by its parent.''' + with self._lock: + if self._fd is not None: + # resource tracker was launched before, is it still running? + if self._check_alive(): + # => still alive + return + # => dead, launch it again + os.close(self._fd) + + # Clean-up to avoid dangling processes. + try: + # _pid can be None if this process is a child from another + # python process, which has started the resource_tracker. + if self._pid is not None: + os.waitpid(self._pid, 0) + except ChildProcessError: + # The resource_tracker has already been terminated. + pass + self._fd = None + self._pid = None + + warnings.warn('resource_tracker: process died unexpectedly, ' + 'relaunching. Some resources might leak.') + + fds_to_pass = [] + try: + fds_to_pass.append(sys.stderr.fileno()) + except Exception: + pass + cmd = 'from multiprocessing.resource_tracker import main;main(%d)' + r, w = os.pipe() + try: + fds_to_pass.append(r) + # process will out live us, so no need to wait on pid + exe = spawn.get_executable() + args = [exe] + util._args_from_interpreter_flags() + args += ['-c', cmd % r] + # bpo-33613: Register a signal mask that will block the signals. + # This signal mask will be inherited by the child that is going + # to be spawned and will protect the child from a race condition + # that can make the child die before it registers signal handlers + # for SIGINT and SIGTERM. The mask is unregistered after spawning + # the child. + try: + if _HAVE_SIGMASK: + signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) + pid = util.spawnv_passfds(exe, args, fds_to_pass) + finally: + if _HAVE_SIGMASK: + signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) + except: + os.close(w) + raise + else: + self._fd = w + self._pid = pid + finally: + os.close(r) + + def _check_alive(self): + '''Check that the pipe has not been closed by sending a probe.''' + try: + # We cannot use send here as it calls ensure_running, creating + # a cycle. + os.write(self._fd, b'PROBE:0:noop\n') + except OSError: + return False + else: + return True + + def register(self, name, rtype): + '''Register name of resource with resource tracker.''' + self._send('REGISTER', name, rtype) + + def unregister(self, name, rtype): + '''Unregister name of resource with resource tracker.''' + self._send('UNREGISTER', name, rtype) + + def _send(self, cmd, name, rtype): + self.ensure_running() + msg = '{0}:{1}:{2}\n'.format(cmd, name, rtype).encode('ascii') + if len(msg) > 512: + # posix guarantees that writes to a pipe of less than PIPE_BUF + # bytes are atomic, and that PIPE_BUF >= 512 + raise ValueError('msg too long') + nbytes = os.write(self._fd, msg) + assert nbytes == len(msg), "nbytes {0:n} but len(msg) {1:n}".format( + nbytes, len(msg)) + + +_resource_tracker = ResourceTracker() +ensure_running = _resource_tracker.ensure_running +register = _resource_tracker.register +unregister = _resource_tracker.unregister +getfd = _resource_tracker.getfd + +def main(fd): + '''Run resource tracker.''' + # protect the process from ^C and "killall python" etc + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + if _HAVE_SIGMASK: + signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) + + for f in (sys.stdin, sys.stdout): + try: + f.close() + except Exception: + pass + + cache = {rtype: set() for rtype in _CLEANUP_FUNCS.keys()} + try: + # keep track of registered/unregistered resources + with open(fd, 'rb') as f: + for line in f: + try: + cmd, name, rtype = line.strip().decode('ascii').split(':') + cleanup_func = _CLEANUP_FUNCS.get(rtype, None) + if cleanup_func is None: + raise ValueError( + f'Cannot register {name} for automatic cleanup: ' + f'unknown resource type {rtype}') + + if cmd == 'REGISTER': + cache[rtype].add(name) + elif cmd == 'UNREGISTER': + cache[rtype].remove(name) + elif cmd == 'PROBE': + pass + else: + raise RuntimeError('unrecognized command %r' % cmd) + except Exception: + try: + sys.excepthook(*sys.exc_info()) + except: + pass + finally: + # all processes have terminated; cleanup any remaining resources + for rtype, rtype_cache in cache.items(): + if rtype_cache: + try: + warnings.warn('resource_tracker: There appear to be %d ' + 'leaked %s objects to clean up at shutdown' % + (len(rtype_cache), rtype)) + except Exception: + pass + for name in rtype_cache: + # For some reason the process which created and registered this + # resource has failed to unregister it. Presumably it has + # died. We therefore unlink it. + try: + try: + _CLEANUP_FUNCS[rtype](name) + except Exception as e: + warnings.warn('resource_tracker: %r: %s' % (name, e)) + finally: + pass diff --git a/lib/python3.1/multiprocessing/shared_memory.py b/lib/python3.1/multiprocessing/shared_memory.py new file mode 100644 index 0000000000000000000000000000000000000000..9a1e5aa17b87a232a831b5d250b8e0c2f15f574d --- /dev/null +++ b/lib/python3.1/multiprocessing/shared_memory.py @@ -0,0 +1,534 @@ +"""Provides shared memory for direct access across processes. + +The API of this package is currently provisional. Refer to the +documentation for details. +""" + + +__all__ = [ 'SharedMemory', 'ShareableList' ] + + +from functools import partial +import mmap +import os +import errno +import struct +import secrets +import types + +if os.name == "nt": + import _winapi + _USE_POSIX = False +else: + import _posixshmem + _USE_POSIX = True + +from . import resource_tracker + +_O_CREX = os.O_CREAT | os.O_EXCL + +# FreeBSD (and perhaps other BSDs) limit names to 14 characters. +_SHM_SAFE_NAME_LENGTH = 14 + +# Shared memory block name prefix +if _USE_POSIX: + _SHM_NAME_PREFIX = '/psm_' +else: + _SHM_NAME_PREFIX = 'wnsm_' + + +def _make_filename(): + "Create a random filename for the shared memory object." + # number of random bytes to use for name + nbytes = (_SHM_SAFE_NAME_LENGTH - len(_SHM_NAME_PREFIX)) // 2 + assert nbytes >= 2, '_SHM_NAME_PREFIX too long' + name = _SHM_NAME_PREFIX + secrets.token_hex(nbytes) + assert len(name) <= _SHM_SAFE_NAME_LENGTH + return name + + +class SharedMemory: + """Creates a new shared memory block or attaches to an existing + shared memory block. + + Every shared memory block is assigned a unique name. This enables + one process to create a shared memory block with a particular name + so that a different process can attach to that same shared memory + block using that same name. + + As a resource for sharing data across processes, shared memory blocks + may outlive the original process that created them. When one process + no longer needs access to a shared memory block that might still be + needed by other processes, the close() method should be called. + When a shared memory block is no longer needed by any process, the + unlink() method should be called to ensure proper cleanup.""" + + # Defaults; enables close() and unlink() to run without errors. + _name = None + _fd = -1 + _mmap = None + _buf = None + _flags = os.O_RDWR + _mode = 0o600 + _prepend_leading_slash = True if _USE_POSIX else False + + def __init__(self, name=None, create=False, size=0): + if not size >= 0: + raise ValueError("'size' must be a positive integer") + if create: + self._flags = _O_CREX | os.O_RDWR + if size == 0: + raise ValueError("'size' must be a positive number different from zero") + if name is None and not self._flags & os.O_EXCL: + raise ValueError("'name' can only be None if create=True") + + if _USE_POSIX: + + # POSIX Shared Memory + + if name is None: + while True: + name = _make_filename() + try: + self._fd = _posixshmem.shm_open( + name, + self._flags, + mode=self._mode + ) + except FileExistsError: + continue + self._name = name + break + else: + name = "/" + name if self._prepend_leading_slash else name + self._fd = _posixshmem.shm_open( + name, + self._flags, + mode=self._mode + ) + self._name = name + try: + if create and size: + os.ftruncate(self._fd, size) + stats = os.fstat(self._fd) + size = stats.st_size + self._mmap = mmap.mmap(self._fd, size) + except OSError: + self.unlink() + raise + + resource_tracker.register(self._name, "shared_memory") + + else: + + # Windows Named Shared Memory + + if create: + while True: + temp_name = _make_filename() if name is None else name + # Create and reserve shared memory block with this name + # until it can be attached to by mmap. + h_map = _winapi.CreateFileMapping( + _winapi.INVALID_HANDLE_VALUE, + _winapi.NULL, + _winapi.PAGE_READWRITE, + (size >> 32) & 0xFFFFFFFF, + size & 0xFFFFFFFF, + temp_name + ) + try: + last_error_code = _winapi.GetLastError() + if last_error_code == _winapi.ERROR_ALREADY_EXISTS: + if name is not None: + raise FileExistsError( + errno.EEXIST, + os.strerror(errno.EEXIST), + name, + _winapi.ERROR_ALREADY_EXISTS + ) + else: + continue + self._mmap = mmap.mmap(-1, size, tagname=temp_name) + finally: + _winapi.CloseHandle(h_map) + self._name = temp_name + break + + else: + self._name = name + # Dynamically determine the existing named shared memory + # block's size which is likely a multiple of mmap.PAGESIZE. + h_map = _winapi.OpenFileMapping( + _winapi.FILE_MAP_READ, + False, + name + ) + try: + p_buf = _winapi.MapViewOfFile( + h_map, + _winapi.FILE_MAP_READ, + 0, + 0, + 0 + ) + finally: + _winapi.CloseHandle(h_map) + try: + size = _winapi.VirtualQuerySize(p_buf) + finally: + _winapi.UnmapViewOfFile(p_buf) + self._mmap = mmap.mmap(-1, size, tagname=name) + + self._size = size + self._buf = memoryview(self._mmap) + + def __del__(self): + try: + self.close() + except OSError: + pass + + def __reduce__(self): + return ( + self.__class__, + ( + self.name, + False, + self.size, + ), + ) + + def __repr__(self): + return f'{self.__class__.__name__}({self.name!r}, size={self.size})' + + @property + def buf(self): + "A memoryview of contents of the shared memory block." + return self._buf + + @property + def name(self): + "Unique name that identifies the shared memory block." + reported_name = self._name + if _USE_POSIX and self._prepend_leading_slash: + if self._name.startswith("/"): + reported_name = self._name[1:] + return reported_name + + @property + def size(self): + "Size in bytes." + return self._size + + def close(self): + """Closes access to the shared memory from this instance but does + not destroy the shared memory block.""" + if self._buf is not None: + self._buf.release() + self._buf = None + if self._mmap is not None: + self._mmap.close() + self._mmap = None + if _USE_POSIX and self._fd >= 0: + os.close(self._fd) + self._fd = -1 + + def unlink(self): + """Requests that the underlying shared memory block be destroyed. + + In order to ensure proper cleanup of resources, unlink should be + called once (and only once) across all processes which have access + to the shared memory block.""" + if _USE_POSIX and self._name: + _posixshmem.shm_unlink(self._name) + resource_tracker.unregister(self._name, "shared_memory") + + +_encoding = "utf8" + +class ShareableList: + """Pattern for a mutable list-like object shareable via a shared + memory block. It differs from the built-in list type in that these + lists can not change their overall length (i.e. no append, insert, + etc.) + + Because values are packed into a memoryview as bytes, the struct + packing format for any storable value must require no more than 8 + characters to describe its format.""" + + # The shared memory area is organized as follows: + # - 8 bytes: number of items (N) as a 64-bit integer + # - (N + 1) * 8 bytes: offsets of each element from the start of the + # data area + # - K bytes: the data area storing item values (with encoding and size + # depending on their respective types) + # - N * 8 bytes: `struct` format string for each element + # - N bytes: index into _back_transforms_mapping for each element + # (for reconstructing the corresponding Python value) + _types_mapping = { + int: "q", + float: "d", + bool: "xxxxxxx?", + str: "%ds", + bytes: "%ds", + None.__class__: "xxxxxx?x", + } + _alignment = 8 + _back_transforms_mapping = { + 0: lambda value: value, # int, float, bool + 1: lambda value: value.rstrip(b'\x00').decode(_encoding), # str + 2: lambda value: value.rstrip(b'\x00'), # bytes + 3: lambda _value: None, # None + } + + @staticmethod + def _extract_recreation_code(value): + """Used in concert with _back_transforms_mapping to convert values + into the appropriate Python objects when retrieving them from + the list as well as when storing them.""" + if not isinstance(value, (str, bytes, None.__class__)): + return 0 + elif isinstance(value, str): + return 1 + elif isinstance(value, bytes): + return 2 + else: + return 3 # NoneType + + def __init__(self, sequence=None, *, name=None): + if name is None or sequence is not None: + sequence = sequence or () + _formats = [ + self._types_mapping[type(item)] + if not isinstance(item, (str, bytes)) + else self._types_mapping[type(item)] % ( + self._alignment * (len(item) // self._alignment + 1), + ) + for item in sequence + ] + self._list_len = len(_formats) + assert sum(len(fmt) <= 8 for fmt in _formats) == self._list_len + offset = 0 + # The offsets of each list element into the shared memory's + # data area (0 meaning the start of the data area, not the start + # of the shared memory area). + self._allocated_offsets = [0] + for fmt in _formats: + offset += self._alignment if fmt[-1] != "s" else int(fmt[:-1]) + self._allocated_offsets.append(offset) + _recreation_codes = [ + self._extract_recreation_code(item) for item in sequence + ] + requested_size = struct.calcsize( + "q" + self._format_size_metainfo + + "".join(_formats) + + self._format_packing_metainfo + + self._format_back_transform_codes + ) + + self.shm = SharedMemory(name, create=True, size=requested_size) + else: + self.shm = SharedMemory(name) + + if sequence is not None: + _enc = _encoding + struct.pack_into( + "q" + self._format_size_metainfo, + self.shm.buf, + 0, + self._list_len, + *(self._allocated_offsets) + ) + struct.pack_into( + "".join(_formats), + self.shm.buf, + self._offset_data_start, + *(v.encode(_enc) if isinstance(v, str) else v for v in sequence) + ) + struct.pack_into( + self._format_packing_metainfo, + self.shm.buf, + self._offset_packing_formats, + *(v.encode(_enc) for v in _formats) + ) + struct.pack_into( + self._format_back_transform_codes, + self.shm.buf, + self._offset_back_transform_codes, + *(_recreation_codes) + ) + + else: + self._list_len = len(self) # Obtains size from offset 0 in buffer. + self._allocated_offsets = list( + struct.unpack_from( + self._format_size_metainfo, + self.shm.buf, + 1 * 8 + ) + ) + + def _get_packing_format(self, position): + "Gets the packing format for a single value stored in the list." + position = position if position >= 0 else position + self._list_len + if (position >= self._list_len) or (self._list_len < 0): + raise IndexError("Requested position out of range.") + + v = struct.unpack_from( + "8s", + self.shm.buf, + self._offset_packing_formats + position * 8 + )[0] + fmt = v.rstrip(b'\x00') + fmt_as_str = fmt.decode(_encoding) + + return fmt_as_str + + def _get_back_transform(self, position): + "Gets the back transformation function for a single value." + + if (position >= self._list_len) or (self._list_len < 0): + raise IndexError("Requested position out of range.") + + transform_code = struct.unpack_from( + "b", + self.shm.buf, + self._offset_back_transform_codes + position + )[0] + transform_function = self._back_transforms_mapping[transform_code] + + return transform_function + + def _set_packing_format_and_transform(self, position, fmt_as_str, value): + """Sets the packing format and back transformation code for a + single value in the list at the specified position.""" + + if (position >= self._list_len) or (self._list_len < 0): + raise IndexError("Requested position out of range.") + + struct.pack_into( + "8s", + self.shm.buf, + self._offset_packing_formats + position * 8, + fmt_as_str.encode(_encoding) + ) + + transform_code = self._extract_recreation_code(value) + struct.pack_into( + "b", + self.shm.buf, + self._offset_back_transform_codes + position, + transform_code + ) + + def __getitem__(self, position): + position = position if position >= 0 else position + self._list_len + try: + offset = self._offset_data_start + self._allocated_offsets[position] + (v,) = struct.unpack_from( + self._get_packing_format(position), + self.shm.buf, + offset + ) + except IndexError: + raise IndexError("index out of range") + + back_transform = self._get_back_transform(position) + v = back_transform(v) + + return v + + def __setitem__(self, position, value): + position = position if position >= 0 else position + self._list_len + try: + item_offset = self._allocated_offsets[position] + offset = self._offset_data_start + item_offset + current_format = self._get_packing_format(position) + except IndexError: + raise IndexError("assignment index out of range") + + if not isinstance(value, (str, bytes)): + new_format = self._types_mapping[type(value)] + encoded_value = value + else: + allocated_length = self._allocated_offsets[position + 1] - item_offset + + encoded_value = (value.encode(_encoding) + if isinstance(value, str) else value) + if len(encoded_value) > allocated_length: + raise ValueError("bytes/str item exceeds available storage") + if current_format[-1] == "s": + new_format = current_format + else: + new_format = self._types_mapping[str] % ( + allocated_length, + ) + + self._set_packing_format_and_transform( + position, + new_format, + value + ) + struct.pack_into(new_format, self.shm.buf, offset, encoded_value) + + def __reduce__(self): + return partial(self.__class__, name=self.shm.name), () + + def __len__(self): + return struct.unpack_from("q", self.shm.buf, 0)[0] + + def __repr__(self): + return f'{self.__class__.__name__}({list(self)}, name={self.shm.name!r})' + + @property + def format(self): + "The struct packing format used by all currently stored items." + return "".join( + self._get_packing_format(i) for i in range(self._list_len) + ) + + @property + def _format_size_metainfo(self): + "The struct packing format used for the items' storage offsets." + return "q" * (self._list_len + 1) + + @property + def _format_packing_metainfo(self): + "The struct packing format used for the items' packing formats." + return "8s" * self._list_len + + @property + def _format_back_transform_codes(self): + "The struct packing format used for the items' back transforms." + return "b" * self._list_len + + @property + def _offset_data_start(self): + # - 8 bytes for the list length + # - (N + 1) * 8 bytes for the element offsets + return (self._list_len + 2) * 8 + + @property + def _offset_packing_formats(self): + return self._offset_data_start + self._allocated_offsets[-1] + + @property + def _offset_back_transform_codes(self): + return self._offset_packing_formats + self._list_len * 8 + + def count(self, value): + "L.count(value) -> integer -- return number of occurrences of value." + + return sum(value == entry for entry in self) + + def index(self, value): + """L.index(value) -> integer -- return first index of value. + Raises ValueError if the value is not present.""" + + for position, entry in enumerate(self): + if value == entry: + return position + else: + raise ValueError(f"{value!r} not in this container") + + __class_getitem__ = classmethod(types.GenericAlias) diff --git a/lib/python3.1/multiprocessing/sharedctypes.py b/lib/python3.1/multiprocessing/sharedctypes.py new file mode 100644 index 0000000000000000000000000000000000000000..6071707027bea46dccfc4d92fb3195060faf07ce --- /dev/null +++ b/lib/python3.1/multiprocessing/sharedctypes.py @@ -0,0 +1,240 @@ +# +# Module which supports allocation of ctypes objects from shared memory +# +# multiprocessing/sharedctypes.py +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +import ctypes +import weakref + +from . import heap +from . import get_context + +from .context import reduction, assert_spawning +_ForkingPickler = reduction.ForkingPickler + +__all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized'] + +# +# +# + +typecode_to_type = { + 'c': ctypes.c_char, 'u': ctypes.c_wchar, + 'b': ctypes.c_byte, 'B': ctypes.c_ubyte, + 'h': ctypes.c_short, 'H': ctypes.c_ushort, + 'i': ctypes.c_int, 'I': ctypes.c_uint, + 'l': ctypes.c_long, 'L': ctypes.c_ulong, + 'q': ctypes.c_longlong, 'Q': ctypes.c_ulonglong, + 'f': ctypes.c_float, 'd': ctypes.c_double + } + +# +# +# + +def _new_value(type_): + size = ctypes.sizeof(type_) + wrapper = heap.BufferWrapper(size) + return rebuild_ctype(type_, wrapper, None) + +def RawValue(typecode_or_type, *args): + ''' + Returns a ctypes object allocated from shared memory + ''' + type_ = typecode_to_type.get(typecode_or_type, typecode_or_type) + obj = _new_value(type_) + ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj)) + obj.__init__(*args) + return obj + +def RawArray(typecode_or_type, size_or_initializer): + ''' + Returns a ctypes array allocated from shared memory + ''' + type_ = typecode_to_type.get(typecode_or_type, typecode_or_type) + if isinstance(size_or_initializer, int): + type_ = type_ * size_or_initializer + obj = _new_value(type_) + ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj)) + return obj + else: + type_ = type_ * len(size_or_initializer) + result = _new_value(type_) + result.__init__(*size_or_initializer) + return result + +def Value(typecode_or_type, *args, lock=True, ctx=None): + ''' + Return a synchronization wrapper for a Value + ''' + obj = RawValue(typecode_or_type, *args) + if lock is False: + return obj + if lock in (True, None): + ctx = ctx or get_context() + lock = ctx.RLock() + if not hasattr(lock, 'acquire'): + raise AttributeError("%r has no method 'acquire'" % lock) + return synchronized(obj, lock, ctx=ctx) + +def Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None): + ''' + Return a synchronization wrapper for a RawArray + ''' + obj = RawArray(typecode_or_type, size_or_initializer) + if lock is False: + return obj + if lock in (True, None): + ctx = ctx or get_context() + lock = ctx.RLock() + if not hasattr(lock, 'acquire'): + raise AttributeError("%r has no method 'acquire'" % lock) + return synchronized(obj, lock, ctx=ctx) + +def copy(obj): + new_obj = _new_value(type(obj)) + ctypes.pointer(new_obj)[0] = obj + return new_obj + +def synchronized(obj, lock=None, ctx=None): + assert not isinstance(obj, SynchronizedBase), 'object already synchronized' + ctx = ctx or get_context() + + if isinstance(obj, ctypes._SimpleCData): + return Synchronized(obj, lock, ctx) + elif isinstance(obj, ctypes.Array): + if obj._type_ is ctypes.c_char: + return SynchronizedString(obj, lock, ctx) + return SynchronizedArray(obj, lock, ctx) + else: + cls = type(obj) + try: + scls = class_cache[cls] + except KeyError: + names = [field[0] for field in cls._fields_] + d = {name: make_property(name) for name in names} + classname = 'Synchronized' + cls.__name__ + scls = class_cache[cls] = type(classname, (SynchronizedBase,), d) + return scls(obj, lock, ctx) + +# +# Functions for pickling/unpickling +# + +def reduce_ctype(obj): + assert_spawning(obj) + if isinstance(obj, ctypes.Array): + return rebuild_ctype, (obj._type_, obj._wrapper, obj._length_) + else: + return rebuild_ctype, (type(obj), obj._wrapper, None) + +def rebuild_ctype(type_, wrapper, length): + if length is not None: + type_ = type_ * length + _ForkingPickler.register(type_, reduce_ctype) + buf = wrapper.create_memoryview() + obj = type_.from_buffer(buf) + obj._wrapper = wrapper + return obj + +# +# Function to create properties +# + +def make_property(name): + try: + return prop_cache[name] + except KeyError: + d = {} + exec(template % ((name,)*7), d) + prop_cache[name] = d[name] + return d[name] + +template = ''' +def get%s(self): + self.acquire() + try: + return self._obj.%s + finally: + self.release() +def set%s(self, value): + self.acquire() + try: + self._obj.%s = value + finally: + self.release() +%s = property(get%s, set%s) +''' + +prop_cache = {} +class_cache = weakref.WeakKeyDictionary() + +# +# Synchronized wrappers +# + +class SynchronizedBase(object): + + def __init__(self, obj, lock=None, ctx=None): + self._obj = obj + if lock: + self._lock = lock + else: + ctx = ctx or get_context(force=True) + self._lock = ctx.RLock() + self.acquire = self._lock.acquire + self.release = self._lock.release + + def __enter__(self): + return self._lock.__enter__() + + def __exit__(self, *args): + return self._lock.__exit__(*args) + + def __reduce__(self): + assert_spawning(self) + return synchronized, (self._obj, self._lock) + + def get_obj(self): + return self._obj + + def get_lock(self): + return self._lock + + def __repr__(self): + return '<%s wrapper for %s>' % (type(self).__name__, self._obj) + + +class Synchronized(SynchronizedBase): + value = make_property('value') + + +class SynchronizedArray(SynchronizedBase): + + def __len__(self): + return len(self._obj) + + def __getitem__(self, i): + with self: + return self._obj[i] + + def __setitem__(self, i, value): + with self: + self._obj[i] = value + + def __getslice__(self, start, stop): + with self: + return self._obj[start:stop] + + def __setslice__(self, start, stop, values): + with self: + self._obj[start:stop] = values + + +class SynchronizedString(SynchronizedArray): + value = make_property('value') + raw = make_property('raw') diff --git a/lib/python3.1/multiprocessing/spawn.py b/lib/python3.1/multiprocessing/spawn.py new file mode 100644 index 0000000000000000000000000000000000000000..7cc129e2610761f3868523030ea4fa60ec9d844b --- /dev/null +++ b/lib/python3.1/multiprocessing/spawn.py @@ -0,0 +1,297 @@ +# +# Code used to start processes when using the spawn or forkserver +# start methods. +# +# multiprocessing/spawn.py +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +import os +import sys +import runpy +import types + +from . import get_start_method, set_start_method +from . import process +from .context import reduction +from . import util + +__all__ = ['_main', 'freeze_support', 'set_executable', 'get_executable', + 'get_preparation_data', 'get_command_line', 'import_main_path'] + +# +# _python_exe is the assumed path to the python executable. +# People embedding Python want to modify it. +# + +if sys.platform != 'win32': + WINEXE = False + WINSERVICE = False +else: + WINEXE = getattr(sys, 'frozen', False) + WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") + +if WINSERVICE: + _python_exe = os.path.join(sys.exec_prefix, 'python.exe') +else: + _python_exe = sys.executable + +def set_executable(exe): + global _python_exe + _python_exe = exe + +def get_executable(): + return _python_exe + +# +# +# + +def is_forking(argv): + ''' + Return whether commandline indicates we are forking + ''' + if len(argv) >= 2 and argv[1] == '--multiprocessing-fork': + return True + else: + return False + + +def freeze_support(): + ''' + Run code for process object if this in not the main process + ''' + if is_forking(sys.argv): + kwds = {} + for arg in sys.argv[2:]: + name, value = arg.split('=') + if value == 'None': + kwds[name] = None + else: + kwds[name] = int(value) + spawn_main(**kwds) + sys.exit() + + +def get_command_line(**kwds): + ''' + Returns prefix of command line used for spawning a child process + ''' + if getattr(sys, 'frozen', False): + return ([sys.executable, '--multiprocessing-fork'] + + ['%s=%r' % item for item in kwds.items()]) + else: + prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)' + prog %= ', '.join('%s=%r' % item for item in kwds.items()) + opts = util._args_from_interpreter_flags() + return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork'] + + +def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None): + ''' + Run code specified by data received over pipe + ''' + assert is_forking(sys.argv), "Not forking" + if sys.platform == 'win32': + import msvcrt + import _winapi + + if parent_pid is not None: + source_process = _winapi.OpenProcess( + _winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE, + False, parent_pid) + else: + source_process = None + new_handle = reduction.duplicate(pipe_handle, + source_process=source_process) + fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY) + parent_sentinel = source_process + else: + from . import resource_tracker + resource_tracker._resource_tracker._fd = tracker_fd + fd = pipe_handle + parent_sentinel = os.dup(pipe_handle) + exitcode = _main(fd, parent_sentinel) + sys.exit(exitcode) + + +def _main(fd, parent_sentinel): + with os.fdopen(fd, 'rb', closefd=True) as from_parent: + process.current_process()._inheriting = True + try: + preparation_data = reduction.pickle.load(from_parent) + prepare(preparation_data) + self = reduction.pickle.load(from_parent) + finally: + del process.current_process()._inheriting + return self._bootstrap(parent_sentinel) + + +def _check_not_importing_main(): + if getattr(process.current_process(), '_inheriting', False): + raise RuntimeError(''' + An attempt has been made to start a new process before the + current process has finished its bootstrapping phase. + + This probably means that you are not using fork to start your + child processes and you have forgotten to use the proper idiom + in the main module: + + if __name__ == '__main__': + freeze_support() + ... + + The "freeze_support()" line can be omitted if the program + is not going to be frozen to produce an executable.''') + + +def get_preparation_data(name): + ''' + Return info about parent needed by child to unpickle process object + ''' + _check_not_importing_main() + d = dict( + log_to_stderr=util._log_to_stderr, + authkey=process.current_process().authkey, + ) + + if util._logger is not None: + d['log_level'] = util._logger.getEffectiveLevel() + + sys_path=sys.path.copy() + try: + i = sys_path.index('') + except ValueError: + pass + else: + sys_path[i] = process.ORIGINAL_DIR + + d.update( + name=name, + sys_path=sys_path, + sys_argv=sys.argv, + orig_dir=process.ORIGINAL_DIR, + dir=os.getcwd(), + start_method=get_start_method(), + ) + + # Figure out whether to initialise main in the subprocess as a module + # or through direct execution (or to leave it alone entirely) + main_module = sys.modules['__main__'] + main_mod_name = getattr(main_module.__spec__, "name", None) + if main_mod_name is not None: + d['init_main_from_name'] = main_mod_name + elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE): + main_path = getattr(main_module, '__file__', None) + if main_path is not None: + if (not os.path.isabs(main_path) and + process.ORIGINAL_DIR is not None): + main_path = os.path.join(process.ORIGINAL_DIR, main_path) + d['init_main_from_path'] = os.path.normpath(main_path) + + return d + +# +# Prepare current process +# + +old_main_modules = [] + +def prepare(data): + ''' + Try to get current process ready to unpickle process object + ''' + if 'name' in data: + process.current_process().name = data['name'] + + if 'authkey' in data: + process.current_process().authkey = data['authkey'] + + if 'log_to_stderr' in data and data['log_to_stderr']: + util.log_to_stderr() + + if 'log_level' in data: + util.get_logger().setLevel(data['log_level']) + + if 'sys_path' in data: + sys.path = data['sys_path'] + + if 'sys_argv' in data: + sys.argv = data['sys_argv'] + + if 'dir' in data: + os.chdir(data['dir']) + + if 'orig_dir' in data: + process.ORIGINAL_DIR = data['orig_dir'] + + if 'start_method' in data: + set_start_method(data['start_method'], force=True) + + if 'init_main_from_name' in data: + _fixup_main_from_name(data['init_main_from_name']) + elif 'init_main_from_path' in data: + _fixup_main_from_path(data['init_main_from_path']) + +# Multiprocessing module helpers to fix up the main module in +# spawned subprocesses +def _fixup_main_from_name(mod_name): + # __main__.py files for packages, directories, zip archives, etc, run + # their "main only" code unconditionally, so we don't even try to + # populate anything in __main__, nor do we make any changes to + # __main__ attributes + current_main = sys.modules['__main__'] + if mod_name == "__main__" or mod_name.endswith(".__main__"): + return + + # If this process was forked, __main__ may already be populated + if getattr(current_main.__spec__, "name", None) == mod_name: + return + + # Otherwise, __main__ may contain some non-main code where we need to + # support unpickling it properly. We rerun it as __mp_main__ and make + # the normal __main__ an alias to that + old_main_modules.append(current_main) + main_module = types.ModuleType("__mp_main__") + main_content = runpy.run_module(mod_name, + run_name="__mp_main__", + alter_sys=True) + main_module.__dict__.update(main_content) + sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module + + +def _fixup_main_from_path(main_path): + # If this process was forked, __main__ may already be populated + current_main = sys.modules['__main__'] + + # Unfortunately, the main ipython launch script historically had no + # "if __name__ == '__main__'" guard, so we work around that + # by treating it like a __main__.py file + # See https://github.com/ipython/ipython/issues/4698 + main_name = os.path.splitext(os.path.basename(main_path))[0] + if main_name == 'ipython': + return + + # Otherwise, if __file__ already has the setting we expect, + # there's nothing more to do + if getattr(current_main, '__file__', None) == main_path: + return + + # If the parent process has sent a path through rather than a module + # name we assume it is an executable script that may contain + # non-main code that needs to be executed + old_main_modules.append(current_main) + main_module = types.ModuleType("__mp_main__") + main_content = runpy.run_path(main_path, + run_name="__mp_main__") + main_module.__dict__.update(main_content) + sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module + + +def import_main_path(main_path): + ''' + Set sys.modules['__main__'] to module at main_path + ''' + _fixup_main_from_path(main_path) diff --git a/lib/python3.1/multiprocessing/synchronize.py b/lib/python3.1/multiprocessing/synchronize.py new file mode 100644 index 0000000000000000000000000000000000000000..d0be48f1fd7a8f14b935f74d89c168d831b84dbc --- /dev/null +++ b/lib/python3.1/multiprocessing/synchronize.py @@ -0,0 +1,394 @@ +# +# Module implementing synchronization primitives +# +# multiprocessing/synchronize.py +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +__all__ = [ + 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event' + ] + +import threading +import sys +import tempfile +import _multiprocessing +import time + +from . import context +from . import process +from . import util + +# Try to import the mp.synchronize module cleanly, if it fails +# raise ImportError for platforms lacking a working sem_open implementation. +# See issue 3770 +try: + from _multiprocessing import SemLock, sem_unlink +except (ImportError): + raise ImportError("This platform lacks a functioning sem_open" + + " implementation, therefore, the required" + + " synchronization primitives needed will not" + + " function, see issue 3770.") + +# +# Constants +# + +RECURSIVE_MUTEX, SEMAPHORE = list(range(2)) +SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX + +# +# Base class for semaphores and mutexes; wraps `_multiprocessing.SemLock` +# + +class SemLock(object): + + _rand = tempfile._RandomNameSequence() + + def __init__(self, kind, value, maxvalue, *, ctx): + if ctx is None: + ctx = context._default_context.get_context() + name = ctx.get_start_method() + unlink_now = sys.platform == 'win32' or name == 'fork' + for i in range(100): + try: + sl = self._semlock = _multiprocessing.SemLock( + kind, value, maxvalue, self._make_name(), + unlink_now) + except FileExistsError: + pass + else: + break + else: + raise FileExistsError('cannot find name for semaphore') + + util.debug('created semlock with handle %s' % sl.handle) + self._make_methods() + + if sys.platform != 'win32': + def _after_fork(obj): + obj._semlock._after_fork() + util.register_after_fork(self, _after_fork) + + if self._semlock.name is not None: + # We only get here if we are on Unix with forking + # disabled. When the object is garbage collected or the + # process shuts down we unlink the semaphore name + from .resource_tracker import register + register(self._semlock.name, "semaphore") + util.Finalize(self, SemLock._cleanup, (self._semlock.name,), + exitpriority=0) + + @staticmethod + def _cleanup(name): + from .resource_tracker import unregister + sem_unlink(name) + unregister(name, "semaphore") + + def _make_methods(self): + self.acquire = self._semlock.acquire + self.release = self._semlock.release + + def __enter__(self): + return self._semlock.__enter__() + + def __exit__(self, *args): + return self._semlock.__exit__(*args) + + def __getstate__(self): + context.assert_spawning(self) + sl = self._semlock + if sys.platform == 'win32': + h = context.get_spawning_popen().duplicate_for_child(sl.handle) + else: + h = sl.handle + return (h, sl.kind, sl.maxvalue, sl.name) + + def __setstate__(self, state): + self._semlock = _multiprocessing.SemLock._rebuild(*state) + util.debug('recreated blocker with handle %r' % state[0]) + self._make_methods() + + @staticmethod + def _make_name(): + return '%s-%s' % (process.current_process()._config['semprefix'], + next(SemLock._rand)) + +# +# Semaphore +# + +class Semaphore(SemLock): + + def __init__(self, value=1, *, ctx): + SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx) + + def get_value(self): + return self._semlock._get_value() + + def __repr__(self): + try: + value = self._semlock._get_value() + except Exception: + value = 'unknown' + return '<%s(value=%s)>' % (self.__class__.__name__, value) + +# +# Bounded semaphore +# + +class BoundedSemaphore(Semaphore): + + def __init__(self, value=1, *, ctx): + SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx) + + def __repr__(self): + try: + value = self._semlock._get_value() + except Exception: + value = 'unknown' + return '<%s(value=%s, maxvalue=%s)>' % \ + (self.__class__.__name__, value, self._semlock.maxvalue) + +# +# Non-recursive lock +# + +class Lock(SemLock): + + def __init__(self, *, ctx): + SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx) + + def __repr__(self): + try: + if self._semlock._is_mine(): + name = process.current_process().name + if threading.current_thread().name != 'MainThread': + name += '|' + threading.current_thread().name + elif self._semlock._get_value() == 1: + name = 'None' + elif self._semlock._count() > 0: + name = 'SomeOtherThread' + else: + name = 'SomeOtherProcess' + except Exception: + name = 'unknown' + return '<%s(owner=%s)>' % (self.__class__.__name__, name) + +# +# Recursive lock +# + +class RLock(SemLock): + + def __init__(self, *, ctx): + SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1, ctx=ctx) + + def __repr__(self): + try: + if self._semlock._is_mine(): + name = process.current_process().name + if threading.current_thread().name != 'MainThread': + name += '|' + threading.current_thread().name + count = self._semlock._count() + elif self._semlock._get_value() == 1: + name, count = 'None', 0 + elif self._semlock._count() > 0: + name, count = 'SomeOtherThread', 'nonzero' + else: + name, count = 'SomeOtherProcess', 'nonzero' + except Exception: + name, count = 'unknown', 'unknown' + return '<%s(%s, %s)>' % (self.__class__.__name__, name, count) + +# +# Condition variable +# + +class Condition(object): + + def __init__(self, lock=None, *, ctx): + self._lock = lock or ctx.RLock() + self._sleeping_count = ctx.Semaphore(0) + self._woken_count = ctx.Semaphore(0) + self._wait_semaphore = ctx.Semaphore(0) + self._make_methods() + + def __getstate__(self): + context.assert_spawning(self) + return (self._lock, self._sleeping_count, + self._woken_count, self._wait_semaphore) + + def __setstate__(self, state): + (self._lock, self._sleeping_count, + self._woken_count, self._wait_semaphore) = state + self._make_methods() + + def __enter__(self): + return self._lock.__enter__() + + def __exit__(self, *args): + return self._lock.__exit__(*args) + + def _make_methods(self): + self.acquire = self._lock.acquire + self.release = self._lock.release + + def __repr__(self): + try: + num_waiters = (self._sleeping_count._semlock._get_value() - + self._woken_count._semlock._get_value()) + except Exception: + num_waiters = 'unknown' + return '<%s(%s, %s)>' % (self.__class__.__name__, self._lock, num_waiters) + + def wait(self, timeout=None): + assert self._lock._semlock._is_mine(), \ + 'must acquire() condition before using wait()' + + # indicate that this thread is going to sleep + self._sleeping_count.release() + + # release lock + count = self._lock._semlock._count() + for i in range(count): + self._lock.release() + + try: + # wait for notification or timeout + return self._wait_semaphore.acquire(True, timeout) + finally: + # indicate that this thread has woken + self._woken_count.release() + + # reacquire lock + for i in range(count): + self._lock.acquire() + + def notify(self, n=1): + assert self._lock._semlock._is_mine(), 'lock is not owned' + assert not self._wait_semaphore.acquire( + False), ('notify: Should not have been able to acquire ' + + '_wait_semaphore') + + # to take account of timeouts since last notify*() we subtract + # woken_count from sleeping_count and rezero woken_count + while self._woken_count.acquire(False): + res = self._sleeping_count.acquire(False) + assert res, ('notify: Bug in sleeping_count.acquire' + + '- res should not be False') + + sleepers = 0 + while sleepers < n and self._sleeping_count.acquire(False): + self._wait_semaphore.release() # wake up one sleeper + sleepers += 1 + + if sleepers: + for i in range(sleepers): + self._woken_count.acquire() # wait for a sleeper to wake + + # rezero wait_semaphore in case some timeouts just happened + while self._wait_semaphore.acquire(False): + pass + + def notify_all(self): + self.notify(n=sys.maxsize) + + def wait_for(self, predicate, timeout=None): + result = predicate() + if result: + return result + if timeout is not None: + endtime = time.monotonic() + timeout + else: + endtime = None + waittime = None + while not result: + if endtime is not None: + waittime = endtime - time.monotonic() + if waittime <= 0: + break + self.wait(waittime) + result = predicate() + return result + +# +# Event +# + +class Event(object): + + def __init__(self, *, ctx): + self._cond = ctx.Condition(ctx.Lock()) + self._flag = ctx.Semaphore(0) + + def is_set(self): + with self._cond: + if self._flag.acquire(False): + self._flag.release() + return True + return False + + def set(self): + with self._cond: + self._flag.acquire(False) + self._flag.release() + self._cond.notify_all() + + def clear(self): + with self._cond: + self._flag.acquire(False) + + def wait(self, timeout=None): + with self._cond: + if self._flag.acquire(False): + self._flag.release() + else: + self._cond.wait(timeout) + + if self._flag.acquire(False): + self._flag.release() + return True + return False + +# +# Barrier +# + +class Barrier(threading.Barrier): + + def __init__(self, parties, action=None, timeout=None, *, ctx): + import struct + from .heap import BufferWrapper + wrapper = BufferWrapper(struct.calcsize('i') * 2) + cond = ctx.Condition() + self.__setstate__((parties, action, timeout, cond, wrapper)) + self._state = 0 + self._count = 0 + + def __setstate__(self, state): + (self._parties, self._action, self._timeout, + self._cond, self._wrapper) = state + self._array = self._wrapper.create_memoryview().cast('i') + + def __getstate__(self): + return (self._parties, self._action, self._timeout, + self._cond, self._wrapper) + + @property + def _state(self): + return self._array[0] + + @_state.setter + def _state(self, value): + self._array[0] = value + + @property + def _count(self): + return self._array[1] + + @_count.setter + def _count(self, value): + self._array[1] = value diff --git a/lib/python3.1/multiprocessing/util.py b/lib/python3.1/multiprocessing/util.py new file mode 100644 index 0000000000000000000000000000000000000000..9e07a4e93e597e853fab59f355419fa5ac282fb2 --- /dev/null +++ b/lib/python3.1/multiprocessing/util.py @@ -0,0 +1,489 @@ +# +# Module providing various facilities to other parts of the package +# +# multiprocessing/util.py +# +# Copyright (c) 2006-2008, R Oudkerk +# Licensed to PSF under a Contributor Agreement. +# + +import os +import itertools +import sys +import weakref +import atexit +import threading # we want threading to install it's + # cleanup function before multiprocessing does +from subprocess import _args_from_interpreter_flags + +from . import process + +__all__ = [ + 'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger', + 'log_to_stderr', 'get_temp_dir', 'register_after_fork', + 'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal', + 'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING', + ] + +# +# Logging +# + +NOTSET = 0 +SUBDEBUG = 5 +DEBUG = 10 +INFO = 20 +SUBWARNING = 25 + +LOGGER_NAME = 'multiprocessing' +DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s' + +_logger = None +_log_to_stderr = False + +def sub_debug(msg, *args): + if _logger: + _logger.log(SUBDEBUG, msg, *args) + +def debug(msg, *args): + if _logger: + _logger.log(DEBUG, msg, *args) + +def info(msg, *args): + if _logger: + _logger.log(INFO, msg, *args) + +def sub_warning(msg, *args): + if _logger: + _logger.log(SUBWARNING, msg, *args) + +def get_logger(): + ''' + Returns logger used by multiprocessing + ''' + global _logger + import logging + + logging._acquireLock() + try: + if not _logger: + + _logger = logging.getLogger(LOGGER_NAME) + _logger.propagate = 0 + + # XXX multiprocessing should cleanup before logging + if hasattr(atexit, 'unregister'): + atexit.unregister(_exit_function) + atexit.register(_exit_function) + else: + atexit._exithandlers.remove((_exit_function, (), {})) + atexit._exithandlers.append((_exit_function, (), {})) + + finally: + logging._releaseLock() + + return _logger + +def log_to_stderr(level=None): + ''' + Turn on logging and add a handler which prints to stderr + ''' + global _log_to_stderr + import logging + + logger = get_logger() + formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + + if level: + logger.setLevel(level) + _log_to_stderr = True + return _logger + + +# Abstract socket support + +def _platform_supports_abstract_sockets(): + if sys.platform == "linux": + return True + if hasattr(sys, 'getandroidapilevel'): + return True + return False + + +def is_abstract_socket_namespace(address): + if not address: + return False + if isinstance(address, bytes): + return address[0] == 0 + elif isinstance(address, str): + return address[0] == "\0" + raise TypeError(f'address type of {address!r} unrecognized') + + +abstract_sockets_supported = _platform_supports_abstract_sockets() + +# +# Function returning a temp directory which will be removed on exit +# + +def _remove_temp_dir(rmtree, tempdir): + rmtree(tempdir) + + current_process = process.current_process() + # current_process() can be None if the finalizer is called + # late during Python finalization + if current_process is not None: + current_process._config['tempdir'] = None + +def get_temp_dir(): + # get name of a temp directory which will be automatically cleaned up + tempdir = process.current_process()._config.get('tempdir') + if tempdir is None: + import shutil, tempfile + tempdir = tempfile.mkdtemp(prefix='pymp-') + info('created temp directory %s', tempdir) + # keep a strong reference to shutil.rmtree(), since the finalizer + # can be called late during Python shutdown + Finalize(None, _remove_temp_dir, args=(shutil.rmtree, tempdir), + exitpriority=-100) + process.current_process()._config['tempdir'] = tempdir + return tempdir + +# +# Support for reinitialization of objects when bootstrapping a child process +# + +_afterfork_registry = weakref.WeakValueDictionary() +_afterfork_counter = itertools.count() + +def _run_after_forkers(): + items = list(_afterfork_registry.items()) + items.sort() + for (index, ident, func), obj in items: + try: + func(obj) + except Exception as e: + info('after forker raised exception %s', e) + +def register_after_fork(obj, func): + _afterfork_registry[(next(_afterfork_counter), id(obj), func)] = obj + +# +# Finalization using weakrefs +# + +_finalizer_registry = {} +_finalizer_counter = itertools.count() + + +class Finalize(object): + ''' + Class which supports object finalization using weakrefs + ''' + def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None): + if (exitpriority is not None) and not isinstance(exitpriority,int): + raise TypeError( + "Exitpriority ({0!r}) must be None or int, not {1!s}".format( + exitpriority, type(exitpriority))) + + if obj is not None: + self._weakref = weakref.ref(obj, self) + elif exitpriority is None: + raise ValueError("Without object, exitpriority cannot be None") + + self._callback = callback + self._args = args + self._kwargs = kwargs or {} + self._key = (exitpriority, next(_finalizer_counter)) + self._pid = os.getpid() + + _finalizer_registry[self._key] = self + + def __call__(self, wr=None, + # Need to bind these locally because the globals can have + # been cleared at shutdown + _finalizer_registry=_finalizer_registry, + sub_debug=sub_debug, getpid=os.getpid): + ''' + Run the callback unless it has already been called or cancelled + ''' + try: + del _finalizer_registry[self._key] + except KeyError: + sub_debug('finalizer no longer registered') + else: + if self._pid != getpid(): + sub_debug('finalizer ignored because different process') + res = None + else: + sub_debug('finalizer calling %s with args %s and kwargs %s', + self._callback, self._args, self._kwargs) + res = self._callback(*self._args, **self._kwargs) + self._weakref = self._callback = self._args = \ + self._kwargs = self._key = None + return res + + def cancel(self): + ''' + Cancel finalization of the object + ''' + try: + del _finalizer_registry[self._key] + except KeyError: + pass + else: + self._weakref = self._callback = self._args = \ + self._kwargs = self._key = None + + def still_active(self): + ''' + Return whether this finalizer is still waiting to invoke callback + ''' + return self._key in _finalizer_registry + + def __repr__(self): + try: + obj = self._weakref() + except (AttributeError, TypeError): + obj = None + + if obj is None: + return '<%s object, dead>' % self.__class__.__name__ + + x = '<%s object, callback=%s' % ( + self.__class__.__name__, + getattr(self._callback, '__name__', self._callback)) + if self._args: + x += ', args=' + str(self._args) + if self._kwargs: + x += ', kwargs=' + str(self._kwargs) + if self._key[0] is not None: + x += ', exitpriority=' + str(self._key[0]) + return x + '>' + + +def _run_finalizers(minpriority=None): + ''' + Run all finalizers whose exit priority is not None and at least minpriority + + Finalizers with highest priority are called first; finalizers with + the same priority will be called in reverse order of creation. + ''' + if _finalizer_registry is None: + # This function may be called after this module's globals are + # destroyed. See the _exit_function function in this module for more + # notes. + return + + if minpriority is None: + f = lambda p : p[0] is not None + else: + f = lambda p : p[0] is not None and p[0] >= minpriority + + # Careful: _finalizer_registry may be mutated while this function + # is running (either by a GC run or by another thread). + + # list(_finalizer_registry) should be atomic, while + # list(_finalizer_registry.items()) is not. + keys = [key for key in list(_finalizer_registry) if f(key)] + keys.sort(reverse=True) + + for key in keys: + finalizer = _finalizer_registry.get(key) + # key may have been removed from the registry + if finalizer is not None: + sub_debug('calling %s', finalizer) + try: + finalizer() + except Exception: + import traceback + traceback.print_exc() + + if minpriority is None: + _finalizer_registry.clear() + +# +# Clean up on exit +# + +def is_exiting(): + ''' + Returns true if the process is shutting down + ''' + return _exiting or _exiting is None + +_exiting = False + +def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers, + active_children=process.active_children, + current_process=process.current_process): + # We hold on to references to functions in the arglist due to the + # situation described below, where this function is called after this + # module's globals are destroyed. + + global _exiting + + if not _exiting: + _exiting = True + + info('process shutting down') + debug('running all "atexit" finalizers with priority >= 0') + _run_finalizers(0) + + if current_process() is not None: + # We check if the current process is None here because if + # it's None, any call to ``active_children()`` will raise + # an AttributeError (active_children winds up trying to + # get attributes from util._current_process). One + # situation where this can happen is if someone has + # manipulated sys.modules, causing this module to be + # garbage collected. The destructor for the module type + # then replaces all values in the module dict with None. + # For instance, after setuptools runs a test it replaces + # sys.modules with a copy created earlier. See issues + # #9775 and #15881. Also related: #4106, #9205, and + # #9207. + + for p in active_children(): + if p.daemon: + info('calling terminate() for daemon %s', p.name) + p._popen.terminate() + + for p in active_children(): + info('calling join() for process %s', p.name) + p.join() + + debug('running the remaining "atexit" finalizers') + _run_finalizers() + +atexit.register(_exit_function) + +# +# Some fork aware types +# + +class ForkAwareThreadLock(object): + def __init__(self): + self._lock = threading.Lock() + self.acquire = self._lock.acquire + self.release = self._lock.release + register_after_fork(self, ForkAwareThreadLock._at_fork_reinit) + + def _at_fork_reinit(self): + self._lock._at_fork_reinit() + + def __enter__(self): + return self._lock.__enter__() + + def __exit__(self, *args): + return self._lock.__exit__(*args) + + +class ForkAwareLocal(threading.local): + def __init__(self): + register_after_fork(self, lambda obj : obj.__dict__.clear()) + def __reduce__(self): + return type(self), () + +# +# Close fds except those specified +# + +try: + MAXFD = os.sysconf("SC_OPEN_MAX") +except Exception: + MAXFD = 256 + +def close_all_fds_except(fds): + fds = list(fds) + [-1, MAXFD] + fds.sort() + assert fds[-1] == MAXFD, 'fd too large' + for i in range(len(fds) - 1): + os.closerange(fds[i]+1, fds[i+1]) +# +# Close sys.stdin and replace stdin with os.devnull +# + +def _close_stdin(): + if sys.stdin is None: + return + + try: + sys.stdin.close() + except (OSError, ValueError): + pass + + try: + fd = os.open(os.devnull, os.O_RDONLY) + try: + sys.stdin = open(fd, encoding="utf-8", closefd=False) + except: + os.close(fd) + raise + except (OSError, ValueError): + pass + +# +# Flush standard streams, if any +# + +def _flush_std_streams(): + try: + sys.stdout.flush() + except (AttributeError, ValueError): + pass + try: + sys.stderr.flush() + except (AttributeError, ValueError): + pass + +# +# Start a program with only specified fds kept open +# + +def spawnv_passfds(path, args, passfds): + import _posixsubprocess + passfds = tuple(sorted(map(int, passfds))) + errpipe_read, errpipe_write = os.pipe() + try: + return _posixsubprocess.fork_exec( + args, [os.fsencode(path)], True, passfds, None, None, + -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, + False, False, None, None, None, -1, None) + finally: + os.close(errpipe_read) + os.close(errpipe_write) + + +def close_fds(*fds): + """Close each file descriptor given as an argument""" + for fd in fds: + os.close(fd) + + +def _cleanup_tests(): + """Cleanup multiprocessing resources when multiprocessing tests + completed.""" + + from test import support + + # cleanup multiprocessing + process._cleanup() + + # Stop the ForkServer process if it's running + from multiprocessing import forkserver + forkserver._forkserver._stop() + + # Stop the ResourceTracker process if it's running + from multiprocessing import resource_tracker + resource_tracker._resource_tracker._stop() + + # bpo-37421: Explicitly call _run_finalizers() to remove immediately + # temporary directories created by multiprocessing.util.get_temp_dir(). + _run_finalizers() + support.gc_collect() + + support.reap_children() diff --git a/lib/python3.1/site-packages/README.txt b/lib/python3.1/site-packages/README.txt new file mode 100644 index 0000000000000000000000000000000000000000..273f6251a7f9d9c4142522e9ab1d699a8ab1bad6 --- /dev/null +++ b/lib/python3.1/site-packages/README.txt @@ -0,0 +1,2 @@ +This directory exists so that 3rd party packages can be installed +here. Read the source for site.py for more details. diff --git a/lib/python3.1/site-packages/aiohttp/.hash/_cparser.pxd.hash b/lib/python3.1/site-packages/aiohttp/.hash/_cparser.pxd.hash new file mode 100644 index 0000000000000000000000000000000000000000..3f5cd0e6f720a049ee19149790c808558b492726 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/.hash/_cparser.pxd.hash @@ -0,0 +1 @@ +5276d46021e0e0d7577e0c9155800cbf62932d60a50783fec42aefb63febedec /home/runner/work/aiohttp/aiohttp/aiohttp/_cparser.pxd diff --git a/lib/python3.1/site-packages/aiohttp/.hash/_find_header.pxd.hash b/lib/python3.1/site-packages/aiohttp/.hash/_find_header.pxd.hash new file mode 100644 index 0000000000000000000000000000000000000000..f006c2de5d24a1b5a9c26f83c858127b5e12b07c --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/.hash/_find_header.pxd.hash @@ -0,0 +1 @@ +d067f01423cddb3c442933b5fcc039b18ab651fcec1bc91c577693aafc25cf78 /home/runner/work/aiohttp/aiohttp/aiohttp/_find_header.pxd diff --git a/lib/python3.1/site-packages/aiohttp/.hash/_http_parser.pyx.hash b/lib/python3.1/site-packages/aiohttp/.hash/_http_parser.pyx.hash new file mode 100644 index 0000000000000000000000000000000000000000..d8c2036a4ec28ee081de8fcd49f107bc0a7222d3 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/.hash/_http_parser.pyx.hash @@ -0,0 +1 @@ +d4bd3b3cab898e00c642eaa59b2f7ae5ae5aa1374e698597f7d805a302f23e21 /home/runner/work/aiohttp/aiohttp/aiohttp/_http_parser.pyx diff --git a/lib/python3.1/site-packages/aiohttp/.hash/_http_writer.pyx.hash b/lib/python3.1/site-packages/aiohttp/.hash/_http_writer.pyx.hash new file mode 100644 index 0000000000000000000000000000000000000000..771a6b17aa5bc2ecd8909c2d3c4b10ce769e55d0 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/.hash/_http_writer.pyx.hash @@ -0,0 +1 @@ +f7ab1e2628277b82772d59c1dc3033c13495d769df67b1d1d49b1a474a75dd52 /home/runner/work/aiohttp/aiohttp/aiohttp/_http_writer.pyx diff --git a/lib/python3.1/site-packages/aiohttp/.hash/hdrs.py.hash b/lib/python3.1/site-packages/aiohttp/.hash/hdrs.py.hash new file mode 100644 index 0000000000000000000000000000000000000000..c8d55240e6a55c305c99244966dee5615565716b --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/.hash/hdrs.py.hash @@ -0,0 +1 @@ +dab8f933203eeb245d60f856e542a45b888d5a110094620e4811f90f816628d1 /home/runner/work/aiohttp/aiohttp/aiohttp/hdrs.py diff --git a/lib/python3.1/site-packages/aiohttp/__init__.py b/lib/python3.1/site-packages/aiohttp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0ca220039374191211bac9f67c8927e0eaf87225 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/__init__.py @@ -0,0 +1,278 @@ +__version__ = "3.12.15" + +from typing import TYPE_CHECKING, Tuple + +from . import hdrs as hdrs +from .client import ( + BaseConnector, + ClientConnectionError, + ClientConnectionResetError, + ClientConnectorCertificateError, + ClientConnectorDNSError, + ClientConnectorError, + ClientConnectorSSLError, + ClientError, + ClientHttpProxyError, + ClientOSError, + ClientPayloadError, + ClientProxyConnectionError, + ClientRequest, + ClientResponse, + ClientResponseError, + ClientSession, + ClientSSLError, + ClientTimeout, + ClientWebSocketResponse, + ClientWSTimeout, + ConnectionTimeoutError, + ContentTypeError, + Fingerprint, + InvalidURL, + InvalidUrlClientError, + InvalidUrlRedirectClientError, + NamedPipeConnector, + NonHttpUrlClientError, + NonHttpUrlRedirectClientError, + RedirectClientError, + RequestInfo, + ServerConnectionError, + ServerDisconnectedError, + ServerFingerprintMismatch, + ServerTimeoutError, + SocketTimeoutError, + TCPConnector, + TooManyRedirects, + UnixConnector, + WSMessageTypeError, + WSServerHandshakeError, + request, +) +from .client_middleware_digest_auth import DigestAuthMiddleware +from .client_middlewares import ClientHandlerType, ClientMiddlewareType +from .compression_utils import set_zlib_backend +from .connector import ( + AddrInfoType as AddrInfoType, + SocketFactoryType as SocketFactoryType, +) +from .cookiejar import CookieJar as CookieJar, DummyCookieJar as DummyCookieJar +from .formdata import FormData as FormData +from .helpers import BasicAuth, ChainMapProxy, ETag +from .http import ( + HttpVersion as HttpVersion, + HttpVersion10 as HttpVersion10, + HttpVersion11 as HttpVersion11, + WebSocketError as WebSocketError, + WSCloseCode as WSCloseCode, + WSMessage as WSMessage, + WSMsgType as WSMsgType, +) +from .multipart import ( + BadContentDispositionHeader as BadContentDispositionHeader, + BadContentDispositionParam as BadContentDispositionParam, + BodyPartReader as BodyPartReader, + MultipartReader as MultipartReader, + MultipartWriter as MultipartWriter, + content_disposition_filename as content_disposition_filename, + parse_content_disposition as parse_content_disposition, +) +from .payload import ( + PAYLOAD_REGISTRY as PAYLOAD_REGISTRY, + AsyncIterablePayload as AsyncIterablePayload, + BufferedReaderPayload as BufferedReaderPayload, + BytesIOPayload as BytesIOPayload, + BytesPayload as BytesPayload, + IOBasePayload as IOBasePayload, + JsonPayload as JsonPayload, + Payload as Payload, + StringIOPayload as StringIOPayload, + StringPayload as StringPayload, + TextIOPayload as TextIOPayload, + get_payload as get_payload, + payload_type as payload_type, +) +from .payload_streamer import streamer as streamer +from .resolver import ( + AsyncResolver as AsyncResolver, + DefaultResolver as DefaultResolver, + ThreadedResolver as ThreadedResolver, +) +from .streams import ( + EMPTY_PAYLOAD as EMPTY_PAYLOAD, + DataQueue as DataQueue, + EofStream as EofStream, + FlowControlDataQueue as FlowControlDataQueue, + StreamReader as StreamReader, +) +from .tracing import ( + TraceConfig as TraceConfig, + TraceConnectionCreateEndParams as TraceConnectionCreateEndParams, + TraceConnectionCreateStartParams as TraceConnectionCreateStartParams, + TraceConnectionQueuedEndParams as TraceConnectionQueuedEndParams, + TraceConnectionQueuedStartParams as TraceConnectionQueuedStartParams, + TraceConnectionReuseconnParams as TraceConnectionReuseconnParams, + TraceDnsCacheHitParams as TraceDnsCacheHitParams, + TraceDnsCacheMissParams as TraceDnsCacheMissParams, + TraceDnsResolveHostEndParams as TraceDnsResolveHostEndParams, + TraceDnsResolveHostStartParams as TraceDnsResolveHostStartParams, + TraceRequestChunkSentParams as TraceRequestChunkSentParams, + TraceRequestEndParams as TraceRequestEndParams, + TraceRequestExceptionParams as TraceRequestExceptionParams, + TraceRequestHeadersSentParams as TraceRequestHeadersSentParams, + TraceRequestRedirectParams as TraceRequestRedirectParams, + TraceRequestStartParams as TraceRequestStartParams, + TraceResponseChunkReceivedParams as TraceResponseChunkReceivedParams, +) + +if TYPE_CHECKING: + # At runtime these are lazy-loaded at the bottom of the file. + from .worker import ( + GunicornUVLoopWebWorker as GunicornUVLoopWebWorker, + GunicornWebWorker as GunicornWebWorker, + ) + +__all__: Tuple[str, ...] = ( + "hdrs", + # client + "AddrInfoType", + "BaseConnector", + "ClientConnectionError", + "ClientConnectionResetError", + "ClientConnectorCertificateError", + "ClientConnectorDNSError", + "ClientConnectorError", + "ClientConnectorSSLError", + "ClientError", + "ClientHttpProxyError", + "ClientOSError", + "ClientPayloadError", + "ClientProxyConnectionError", + "ClientResponse", + "ClientRequest", + "ClientResponseError", + "ClientSSLError", + "ClientSession", + "ClientTimeout", + "ClientWebSocketResponse", + "ClientWSTimeout", + "ConnectionTimeoutError", + "ContentTypeError", + "Fingerprint", + "FlowControlDataQueue", + "InvalidURL", + "InvalidUrlClientError", + "InvalidUrlRedirectClientError", + "NonHttpUrlClientError", + "NonHttpUrlRedirectClientError", + "RedirectClientError", + "RequestInfo", + "ServerConnectionError", + "ServerDisconnectedError", + "ServerFingerprintMismatch", + "ServerTimeoutError", + "SocketFactoryType", + "SocketTimeoutError", + "TCPConnector", + "TooManyRedirects", + "UnixConnector", + "NamedPipeConnector", + "WSServerHandshakeError", + "request", + # client_middleware + "ClientMiddlewareType", + "ClientHandlerType", + # cookiejar + "CookieJar", + "DummyCookieJar", + # formdata + "FormData", + # helpers + "BasicAuth", + "ChainMapProxy", + "DigestAuthMiddleware", + "ETag", + "set_zlib_backend", + # http + "HttpVersion", + "HttpVersion10", + "HttpVersion11", + "WSMsgType", + "WSCloseCode", + "WSMessage", + "WebSocketError", + # multipart + "BadContentDispositionHeader", + "BadContentDispositionParam", + "BodyPartReader", + "MultipartReader", + "MultipartWriter", + "content_disposition_filename", + "parse_content_disposition", + # payload + "AsyncIterablePayload", + "BufferedReaderPayload", + "BytesIOPayload", + "BytesPayload", + "IOBasePayload", + "JsonPayload", + "PAYLOAD_REGISTRY", + "Payload", + "StringIOPayload", + "StringPayload", + "TextIOPayload", + "get_payload", + "payload_type", + # payload_streamer + "streamer", + # resolver + "AsyncResolver", + "DefaultResolver", + "ThreadedResolver", + # streams + "DataQueue", + "EMPTY_PAYLOAD", + "EofStream", + "StreamReader", + # tracing + "TraceConfig", + "TraceConnectionCreateEndParams", + "TraceConnectionCreateStartParams", + "TraceConnectionQueuedEndParams", + "TraceConnectionQueuedStartParams", + "TraceConnectionReuseconnParams", + "TraceDnsCacheHitParams", + "TraceDnsCacheMissParams", + "TraceDnsResolveHostEndParams", + "TraceDnsResolveHostStartParams", + "TraceRequestChunkSentParams", + "TraceRequestEndParams", + "TraceRequestExceptionParams", + "TraceRequestHeadersSentParams", + "TraceRequestRedirectParams", + "TraceRequestStartParams", + "TraceResponseChunkReceivedParams", + # workers (imported lazily with __getattr__) + "GunicornUVLoopWebWorker", + "GunicornWebWorker", + "WSMessageTypeError", +) + + +def __dir__() -> Tuple[str, ...]: + return __all__ + ("__doc__",) + + +def __getattr__(name: str) -> object: + global GunicornUVLoopWebWorker, GunicornWebWorker + + # Importing gunicorn takes a long time (>100ms), so only import if actually needed. + if name in ("GunicornUVLoopWebWorker", "GunicornWebWorker"): + try: + from .worker import GunicornUVLoopWebWorker as guv, GunicornWebWorker as gw + except ImportError: + return None + + GunicornUVLoopWebWorker = guv # type: ignore[misc] + GunicornWebWorker = gw # type: ignore[misc] + return guv if name == "GunicornUVLoopWebWorker" else gw + + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3896e4aec8db79e8057a0eaa63f98572c5e2eb3a Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/__init__.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/_cookie_helpers.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/_cookie_helpers.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7da1082dc7db40db2d3e539d2a3a6ffaa80649ad Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/_cookie_helpers.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/abc.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/abc.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38e509e03100f674616cf74bc0613f91419ffa02 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/abc.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/base_protocol.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/base_protocol.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d18d4a7e0496ddcbd3ccf2cd725e0d4cae694224 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/base_protocol.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/client.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/client.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..19760eccc509607b5b0171eb27529449725b04e1 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/client.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/client_exceptions.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/client_exceptions.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cea6bd874fa0267d4632493f89496850fd396905 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/client_exceptions.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/client_middleware_digest_auth.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/client_middleware_digest_auth.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38c70950d5a5db8e775b2b7151e44f846640d83e Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/client_middleware_digest_auth.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/client_middlewares.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/client_middlewares.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..522bb9dbe2c5e9f87859b7f7023033e896874feb Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/client_middlewares.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/client_proto.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/client_proto.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..09226fcbebda6fc3aece639d1b7be4fc98793be5 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/client_proto.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/client_reqrep.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/client_reqrep.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9faefa75582364d36708c4d8c52258ee47316873 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/client_reqrep.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/client_ws.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/client_ws.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7506038a8c05c9352db1b83fca6c1edf5b75ea7d Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/client_ws.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/compression_utils.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/compression_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..85b2cb17e4a9007dcaf81a58714de24579f2d1be Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/compression_utils.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/connector.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/connector.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a42834bdc38af55a8872586aebadf64b969a2c15 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/connector.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/cookiejar.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/cookiejar.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bad08731c36816b7d154a1b39d44e732d33efaac Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/cookiejar.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/formdata.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/formdata.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..69d26de4b64e16f40d48197a2c8d0852510791c6 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/formdata.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/hdrs.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/hdrs.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6977badfccb78f112b0912478b8270de8f3e4ceb Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/hdrs.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/helpers.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/helpers.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb9b8bb6b3e74b2bf3a7e70459da37aaa40840ba Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/helpers.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/http.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/http.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61d19931dba2c1eb193056c8af85a75b7b8e01f3 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/http.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/http_exceptions.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/http_exceptions.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57907f02ccc86c9043d2f3ef9a702a6ac0ea0a27 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/http_exceptions.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/http_parser.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/http_parser.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08914f9f149fa114dedb10687890ac9380abbd9a Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/http_parser.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/http_websocket.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/http_websocket.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..35c785b696942860b29113d00bd7b67bdaf82fb5 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/http_websocket.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/http_writer.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/http_writer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..faec76c14121a5240ea6f03740fb60127d98a00e Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/http_writer.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/log.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/log.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f542f88ee82511016ec91bc2374fe934ff959e15 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/log.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/multipart.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/multipart.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..80b47472e888e35ca0fe85197e7ac7b8a003459a Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/multipart.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/payload.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/payload.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..17bd171223dc59e82aada1e00c95411bda3db1ef Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/payload.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/payload_streamer.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/payload_streamer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b9a04b2dd04045f83ee2988047f78ec0e52dfdb5 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/payload_streamer.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/pytest_plugin.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/pytest_plugin.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4be6eb457133af593645d86fabcd9d292720ec34 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/pytest_plugin.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/resolver.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/resolver.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3c65eea29739ce7d94b740a92c693d2ffe18d67d Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/resolver.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/streams.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/streams.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2774fd17511227e77b36728810ee806ac7bda588 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/streams.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/tcp_helpers.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/tcp_helpers.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..501b7363c0ec53afe075e961907f961f2c4c57a6 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/tcp_helpers.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/test_utils.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/test_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7765b20dc61eb07c8cf386e8f269e94e54a8a9a1 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/test_utils.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/tracing.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/tracing.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8595074119450329bd4409ef8309a99262843601 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/tracing.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/typedefs.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/typedefs.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..45cf85f69da3052eeb47191e0d966395eb425b3b Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/typedefs.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1e52d173ff68d539c3387c118e2bb7d80aa42064 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web_app.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web_app.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d8497c6bb662a16e6b9c92959de0f888c2166a86 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web_app.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web_exceptions.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web_exceptions.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..36335cb0e5dc213912f2500136efc9dd5947146b Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web_exceptions.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web_fileresponse.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web_fileresponse.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..92580cfb19292ca578acc4c34292f4b2d54e6f14 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web_fileresponse.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web_log.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web_log.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..04438288cd06f5faf8f7337f6a612a050a8bbd9d Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web_log.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web_middlewares.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web_middlewares.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8fa5885973a022cfef68ddc8c37c757886c0508d Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web_middlewares.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web_protocol.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web_protocol.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..25a055e1b9d7e5a0ca46b1b4ba940057a5bb263a Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web_protocol.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web_request.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web_request.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..15571f33f63604848de59891a305e148e8e6bee8 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web_request.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web_response.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web_response.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6330d377eaad8e832189800a122f735ae83243a Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web_response.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web_routedef.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web_routedef.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..affc78035ba29dd4f518a7356b3f03455ef6c4fd Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web_routedef.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web_runner.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web_runner.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..720a2d5bc683d4be06f8f9b58f0de14a4b2652a2 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web_runner.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web_server.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web_server.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2fde001b01588fe7c0a60946dd8903a8d4fcef81 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web_server.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web_urldispatcher.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web_urldispatcher.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5803a866f6f7e2f2ee64f15793826a3dbf34cbbc Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web_urldispatcher.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/web_ws.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/web_ws.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..425f3932d17da23578b2569508608f1bd752861b Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/web_ws.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/__pycache__/worker.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/__pycache__/worker.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cfe11590faa94559f64e7ad205b8bef69c74b167 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/__pycache__/worker.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/_cookie_helpers.py b/lib/python3.1/site-packages/aiohttp/_cookie_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..4e9fc968814efa9a4990a1a40f39dfea7553b6aa --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_cookie_helpers.py @@ -0,0 +1,309 @@ +""" +Internal cookie handling helpers. + +This module contains internal utilities for cookie parsing and manipulation. +These are not part of the public API and may change without notice. +""" + +import re +import sys +from http.cookies import Morsel +from typing import List, Optional, Sequence, Tuple, cast + +from .log import internal_logger + +__all__ = ( + "parse_set_cookie_headers", + "parse_cookie_header", + "preserve_morsel_with_coded_value", +) + +# Cookie parsing constants +# Allow more characters in cookie names to handle real-world cookies +# that don't strictly follow RFC standards (fixes #2683) +# RFC 6265 defines cookie-name token as per RFC 2616 Section 2.2, +# but many servers send cookies with characters like {} [] () etc. +# This makes the cookie parser more tolerant of real-world cookies +# while still providing some validation to catch obviously malformed names. +_COOKIE_NAME_RE = re.compile(r"^[!#$%&\'()*+\-./0-9:<=>?@A-Z\[\]^_`a-z{|}~]+$") +_COOKIE_KNOWN_ATTRS = frozenset( # AKA Morsel._reserved + ( + "path", + "domain", + "max-age", + "expires", + "secure", + "httponly", + "samesite", + "partitioned", + "version", + "comment", + ) +) +_COOKIE_BOOL_ATTRS = frozenset( # AKA Morsel._flags + ("secure", "httponly", "partitioned") +) + +# SimpleCookie's pattern for parsing cookies with relaxed validation +# Based on http.cookies pattern but extended to allow more characters in cookie names +# to handle real-world cookies (fixes #2683) +_COOKIE_PATTERN = re.compile( + r""" + \s* # Optional whitespace at start of cookie + (?P # Start of group 'key' + # aiohttp has extended to include [] for compatibility with real-world cookies + [\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=\[\]]+? # Any word of at least one letter + ) # End of group 'key' + ( # Optional group: there may not be a value. + \s*=\s* # Equal Sign + (?P # Start of group 'val' + "(?:[^\\"]|\\.)*" # Any double-quoted string (properly closed) + | # or + "[^";]* # Unmatched opening quote (differs from SimpleCookie - issue #7993) + | # or + # Special case for "expires" attr - RFC 822, RFC 850, RFC 1036, RFC 1123 + (\w{3,6}day|\w{3}),\s # Day of the week or abbreviated day (with comma) + [\w\d\s-]{9,11}\s[\d:]{8}\s # Date and time in specific format + (GMT|[+-]\d{4}) # Timezone: GMT or RFC 2822 offset like -0000, +0100 + # NOTE: RFC 2822 timezone support is an aiohttp extension + # for issue #4493 - SimpleCookie does NOT support this + | # or + # ANSI C asctime() format: "Wed Jun 9 10:18:14 2021" + # NOTE: This is an aiohttp extension for issue #4327 - SimpleCookie does NOT support this format + \w{3}\s+\w{3}\s+[\s\d]\d\s+\d{2}:\d{2}:\d{2}\s+\d{4} + | # or + [\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=\[\]]* # Any word or empty string + ) # End of group 'val' + )? # End of optional value group + \s* # Any number of spaces. + (\s+|;|$) # Ending either at space, semicolon, or EOS. + """, + re.VERBOSE | re.ASCII, +) + + +def preserve_morsel_with_coded_value(cookie: Morsel[str]) -> Morsel[str]: + """ + Preserve a Morsel's coded_value exactly as received from the server. + + This function ensures that cookie encoding is preserved exactly as sent by + the server, which is critical for compatibility with old servers that have + strict requirements about cookie formats. + + This addresses the issue described in https://github.com/aio-libs/aiohttp/pull/1453 + where Python's SimpleCookie would re-encode cookies, breaking authentication + with certain servers. + + Args: + cookie: A Morsel object from SimpleCookie + + Returns: + A Morsel object with preserved coded_value + + """ + mrsl_val = cast("Morsel[str]", cookie.get(cookie.key, Morsel())) + # We use __setstate__ instead of the public set() API because it allows us to + # bypass validation and set already validated state. This is more stable than + # setting protected attributes directly and unlikely to change since it would + # break pickling. + mrsl_val.__setstate__( # type: ignore[attr-defined] + {"key": cookie.key, "value": cookie.value, "coded_value": cookie.coded_value} + ) + return mrsl_val + + +_unquote_sub = re.compile(r"\\(?:([0-3][0-7][0-7])|(.))").sub + + +def _unquote_replace(m: re.Match[str]) -> str: + """ + Replace function for _unquote_sub regex substitution. + + Handles escaped characters in cookie values: + - Octal sequences are converted to their character representation + - Other escaped characters are unescaped by removing the backslash + """ + if m[1]: + return chr(int(m[1], 8)) + return m[2] + + +def _unquote(value: str) -> str: + """ + Unquote a cookie value. + + Vendored from http.cookies._unquote to ensure compatibility. + + Note: The original implementation checked for None, but we've removed + that check since all callers already ensure the value is not None. + """ + # If there aren't any doublequotes, + # then there can't be any special characters. See RFC 2109. + if len(value) < 2: + return value + if value[0] != '"' or value[-1] != '"': + return value + + # We have to assume that we must decode this string. + # Down to work. + + # Remove the "s + value = value[1:-1] + + # Check for special sequences. Examples: + # \012 --> \n + # \" --> " + # + return _unquote_sub(_unquote_replace, value) + + +def parse_cookie_header(header: str) -> List[Tuple[str, Morsel[str]]]: + """ + Parse a Cookie header according to RFC 6265 Section 5.4. + + Cookie headers contain only name-value pairs separated by semicolons. + There are no attributes in Cookie headers - even names that match + attribute names (like 'path' or 'secure') should be treated as cookies. + + This parser uses the same regex-based approach as parse_set_cookie_headers + to properly handle quoted values that may contain semicolons. + + Args: + header: The Cookie header value to parse + + Returns: + List of (name, Morsel) tuples for compatibility with SimpleCookie.update() + """ + if not header: + return [] + + cookies: List[Tuple[str, Morsel[str]]] = [] + i = 0 + n = len(header) + + while i < n: + # Use the same pattern as parse_set_cookie_headers to find cookies + match = _COOKIE_PATTERN.match(header, i) + if not match: + break + + key = match.group("key") + value = match.group("val") or "" + i = match.end(0) + + # Validate the name + if not key or not _COOKIE_NAME_RE.match(key): + internal_logger.warning("Can not load cookie: Illegal cookie name %r", key) + continue + + # Create new morsel + morsel: Morsel[str] = Morsel() + # Preserve the original value as coded_value (with quotes if present) + # We use __setstate__ instead of the public set() API because it allows us to + # bypass validation and set already validated state. This is more stable than + # setting protected attributes directly and unlikely to change since it would + # break pickling. + morsel.__setstate__( # type: ignore[attr-defined] + {"key": key, "value": _unquote(value), "coded_value": value} + ) + + cookies.append((key, morsel)) + + return cookies + + +def parse_set_cookie_headers(headers: Sequence[str]) -> List[Tuple[str, Morsel[str]]]: + """ + Parse cookie headers using a vendored version of SimpleCookie parsing. + + This implementation is based on SimpleCookie.__parse_string to ensure + compatibility with how SimpleCookie parses cookies, including handling + of malformed cookies with missing semicolons. + + This function is used for both Cookie and Set-Cookie headers in order to be + forgiving. Ideally we would have followed RFC 6265 Section 5.2 (for Cookie + headers) and RFC 6265 Section 4.2.1 (for Set-Cookie headers), but the + real world data makes it impossible since we need to be a bit more forgiving. + + NOTE: This implementation differs from SimpleCookie in handling unmatched quotes. + SimpleCookie will stop parsing when it encounters a cookie value with an unmatched + quote (e.g., 'cookie="value'), causing subsequent cookies to be silently dropped. + This implementation handles unmatched quotes more gracefully to prevent cookie loss. + See https://github.com/aio-libs/aiohttp/issues/7993 + """ + parsed_cookies: List[Tuple[str, Morsel[str]]] = [] + + for header in headers: + if not header: + continue + + # Parse cookie string using SimpleCookie's algorithm + i = 0 + n = len(header) + current_morsel: Optional[Morsel[str]] = None + morsel_seen = False + + while 0 <= i < n: + # Start looking for a cookie + match = _COOKIE_PATTERN.match(header, i) + if not match: + # No more cookies + break + + key, value = match.group("key"), match.group("val") + i = match.end(0) + lower_key = key.lower() + + if key[0] == "$": + if not morsel_seen: + # We ignore attributes which pertain to the cookie + # mechanism as a whole, such as "$Version". + continue + # Process as attribute + if current_morsel is not None: + attr_lower_key = lower_key[1:] + if attr_lower_key in _COOKIE_KNOWN_ATTRS: + current_morsel[attr_lower_key] = value or "" + elif lower_key in _COOKIE_KNOWN_ATTRS: + if not morsel_seen: + # Invalid cookie string - attribute before cookie + break + if lower_key in _COOKIE_BOOL_ATTRS: + # Boolean attribute with any value should be True + if current_morsel is not None: + if lower_key == "partitioned" and sys.version_info < (3, 14): + dict.__setitem__(current_morsel, lower_key, True) + else: + current_morsel[lower_key] = True + elif value is None: + # Invalid cookie string - non-boolean attribute without value + break + elif current_morsel is not None: + # Regular attribute with value + current_morsel[lower_key] = _unquote(value) + elif value is not None: + # This is a cookie name=value pair + # Validate the name + if key in _COOKIE_KNOWN_ATTRS or not _COOKIE_NAME_RE.match(key): + internal_logger.warning( + "Can not load cookies: Illegal cookie name %r", key + ) + current_morsel = None + else: + # Create new morsel + current_morsel = Morsel() + # Preserve the original value as coded_value (with quotes if present) + # We use __setstate__ instead of the public set() API because it allows us to + # bypass validation and set already validated state. This is more stable than + # setting protected attributes directly and unlikely to change since it would + # break pickling. + current_morsel.__setstate__( # type: ignore[attr-defined] + {"key": key, "value": _unquote(value), "coded_value": value} + ) + parsed_cookies.append((key, current_morsel)) + morsel_seen = True + else: + # Invalid cookie string - no value for non-attribute + break + + return parsed_cookies diff --git a/lib/python3.1/site-packages/aiohttp/_cparser.pxd b/lib/python3.1/site-packages/aiohttp/_cparser.pxd new file mode 100644 index 0000000000000000000000000000000000000000..1b3be6d4efb682bca9397da34f8e727b381bc84f --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_cparser.pxd @@ -0,0 +1,158 @@ +from libc.stdint cimport int32_t, uint8_t, uint16_t, uint64_t + + +cdef extern from "llhttp.h": + + struct llhttp__internal_s: + int32_t _index + void* _span_pos0 + void* _span_cb0 + int32_t error + const char* reason + const char* error_pos + void* data + void* _current + uint64_t content_length + uint8_t type + uint8_t method + uint8_t http_major + uint8_t http_minor + uint8_t header_state + uint8_t lenient_flags + uint8_t upgrade + uint8_t finish + uint16_t flags + uint16_t status_code + void* settings + + ctypedef llhttp__internal_s llhttp__internal_t + ctypedef llhttp__internal_t llhttp_t + + ctypedef int (*llhttp_data_cb)(llhttp_t*, const char *at, size_t length) except -1 + ctypedef int (*llhttp_cb)(llhttp_t*) except -1 + + struct llhttp_settings_s: + llhttp_cb on_message_begin + llhttp_data_cb on_url + llhttp_data_cb on_status + llhttp_data_cb on_header_field + llhttp_data_cb on_header_value + llhttp_cb on_headers_complete + llhttp_data_cb on_body + llhttp_cb on_message_complete + llhttp_cb on_chunk_header + llhttp_cb on_chunk_complete + + llhttp_cb on_url_complete + llhttp_cb on_status_complete + llhttp_cb on_header_field_complete + llhttp_cb on_header_value_complete + + ctypedef llhttp_settings_s llhttp_settings_t + + enum llhttp_errno: + HPE_OK, + HPE_INTERNAL, + HPE_STRICT, + HPE_LF_EXPECTED, + HPE_UNEXPECTED_CONTENT_LENGTH, + HPE_CLOSED_CONNECTION, + HPE_INVALID_METHOD, + HPE_INVALID_URL, + HPE_INVALID_CONSTANT, + HPE_INVALID_VERSION, + HPE_INVALID_HEADER_TOKEN, + HPE_INVALID_CONTENT_LENGTH, + HPE_INVALID_CHUNK_SIZE, + HPE_INVALID_STATUS, + HPE_INVALID_EOF_STATE, + HPE_INVALID_TRANSFER_ENCODING, + HPE_CB_MESSAGE_BEGIN, + HPE_CB_HEADERS_COMPLETE, + HPE_CB_MESSAGE_COMPLETE, + HPE_CB_CHUNK_HEADER, + HPE_CB_CHUNK_COMPLETE, + HPE_PAUSED, + HPE_PAUSED_UPGRADE, + HPE_USER + + ctypedef llhttp_errno llhttp_errno_t + + enum llhttp_flags: + F_CHUNKED, + F_CONTENT_LENGTH + + enum llhttp_type: + HTTP_REQUEST, + HTTP_RESPONSE, + HTTP_BOTH + + enum llhttp_method: + HTTP_DELETE, + HTTP_GET, + HTTP_HEAD, + HTTP_POST, + HTTP_PUT, + HTTP_CONNECT, + HTTP_OPTIONS, + HTTP_TRACE, + HTTP_COPY, + HTTP_LOCK, + HTTP_MKCOL, + HTTP_MOVE, + HTTP_PROPFIND, + HTTP_PROPPATCH, + HTTP_SEARCH, + HTTP_UNLOCK, + HTTP_BIND, + HTTP_REBIND, + HTTP_UNBIND, + HTTP_ACL, + HTTP_REPORT, + HTTP_MKACTIVITY, + HTTP_CHECKOUT, + HTTP_MERGE, + HTTP_MSEARCH, + HTTP_NOTIFY, + HTTP_SUBSCRIBE, + HTTP_UNSUBSCRIBE, + HTTP_PATCH, + HTTP_PURGE, + HTTP_MKCALENDAR, + HTTP_LINK, + HTTP_UNLINK, + HTTP_SOURCE, + HTTP_PRI, + HTTP_DESCRIBE, + HTTP_ANNOUNCE, + HTTP_SETUP, + HTTP_PLAY, + HTTP_PAUSE, + HTTP_TEARDOWN, + HTTP_GET_PARAMETER, + HTTP_SET_PARAMETER, + HTTP_REDIRECT, + HTTP_RECORD, + HTTP_FLUSH + + ctypedef llhttp_method llhttp_method_t; + + void llhttp_settings_init(llhttp_settings_t* settings) + void llhttp_init(llhttp_t* parser, llhttp_type type, + const llhttp_settings_t* settings) + + llhttp_errno_t llhttp_execute(llhttp_t* parser, const char* data, size_t len) + + int llhttp_should_keep_alive(const llhttp_t* parser) + + void llhttp_resume_after_upgrade(llhttp_t* parser) + + llhttp_errno_t llhttp_get_errno(const llhttp_t* parser) + const char* llhttp_get_error_reason(const llhttp_t* parser) + const char* llhttp_get_error_pos(const llhttp_t* parser) + + const char* llhttp_method_name(llhttp_method_t method) + + void llhttp_set_lenient_headers(llhttp_t* parser, int enabled) + void llhttp_set_lenient_optional_cr_before_lf(llhttp_t* parser, int enabled) + void llhttp_set_lenient_spaces_after_chunk_size(llhttp_t* parser, int enabled) diff --git a/lib/python3.1/site-packages/aiohttp/_find_header.pxd b/lib/python3.1/site-packages/aiohttp/_find_header.pxd new file mode 100644 index 0000000000000000000000000000000000000000..37a6c37268ee30b182fd77d109688d35d5577c7f --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_find_header.pxd @@ -0,0 +1,2 @@ +cdef extern from "_find_header.h": + int find_header(char *, int) diff --git a/lib/python3.1/site-packages/aiohttp/_headers.pxi b/lib/python3.1/site-packages/aiohttp/_headers.pxi new file mode 100644 index 0000000000000000000000000000000000000000..3744721d4786a6c79b90aa349c8d02fa66204ecc --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_headers.pxi @@ -0,0 +1,83 @@ +# The file is autogenerated from aiohttp/hdrs.py +# Run ./tools/gen.py to update it after the origin changing. + +from . import hdrs +cdef tuple headers = ( + hdrs.ACCEPT, + hdrs.ACCEPT_CHARSET, + hdrs.ACCEPT_ENCODING, + hdrs.ACCEPT_LANGUAGE, + hdrs.ACCEPT_RANGES, + hdrs.ACCESS_CONTROL_ALLOW_CREDENTIALS, + hdrs.ACCESS_CONTROL_ALLOW_HEADERS, + hdrs.ACCESS_CONTROL_ALLOW_METHODS, + hdrs.ACCESS_CONTROL_ALLOW_ORIGIN, + hdrs.ACCESS_CONTROL_EXPOSE_HEADERS, + hdrs.ACCESS_CONTROL_MAX_AGE, + hdrs.ACCESS_CONTROL_REQUEST_HEADERS, + hdrs.ACCESS_CONTROL_REQUEST_METHOD, + hdrs.AGE, + hdrs.ALLOW, + hdrs.AUTHORIZATION, + hdrs.CACHE_CONTROL, + hdrs.CONNECTION, + hdrs.CONTENT_DISPOSITION, + hdrs.CONTENT_ENCODING, + hdrs.CONTENT_LANGUAGE, + hdrs.CONTENT_LENGTH, + hdrs.CONTENT_LOCATION, + hdrs.CONTENT_MD5, + hdrs.CONTENT_RANGE, + hdrs.CONTENT_TRANSFER_ENCODING, + hdrs.CONTENT_TYPE, + hdrs.COOKIE, + hdrs.DATE, + hdrs.DESTINATION, + hdrs.DIGEST, + hdrs.ETAG, + hdrs.EXPECT, + hdrs.EXPIRES, + hdrs.FORWARDED, + hdrs.FROM, + hdrs.HOST, + hdrs.IF_MATCH, + hdrs.IF_MODIFIED_SINCE, + hdrs.IF_NONE_MATCH, + hdrs.IF_RANGE, + hdrs.IF_UNMODIFIED_SINCE, + hdrs.KEEP_ALIVE, + hdrs.LAST_EVENT_ID, + hdrs.LAST_MODIFIED, + hdrs.LINK, + hdrs.LOCATION, + hdrs.MAX_FORWARDS, + hdrs.ORIGIN, + hdrs.PRAGMA, + hdrs.PROXY_AUTHENTICATE, + hdrs.PROXY_AUTHORIZATION, + hdrs.RANGE, + hdrs.REFERER, + hdrs.RETRY_AFTER, + hdrs.SEC_WEBSOCKET_ACCEPT, + hdrs.SEC_WEBSOCKET_EXTENSIONS, + hdrs.SEC_WEBSOCKET_KEY, + hdrs.SEC_WEBSOCKET_KEY1, + hdrs.SEC_WEBSOCKET_PROTOCOL, + hdrs.SEC_WEBSOCKET_VERSION, + hdrs.SERVER, + hdrs.SET_COOKIE, + hdrs.TE, + hdrs.TRAILER, + hdrs.TRANSFER_ENCODING, + hdrs.URI, + hdrs.UPGRADE, + hdrs.USER_AGENT, + hdrs.VARY, + hdrs.VIA, + hdrs.WWW_AUTHENTICATE, + hdrs.WANT_DIGEST, + hdrs.WARNING, + hdrs.X_FORWARDED_FOR, + hdrs.X_FORWARDED_HOST, + hdrs.X_FORWARDED_PROTO, +) diff --git a/lib/python3.1/site-packages/aiohttp/_http_parser.pyx b/lib/python3.1/site-packages/aiohttp/_http_parser.pyx new file mode 100644 index 0000000000000000000000000000000000000000..16893f00e7435f4125d68446a6187791c4604549 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_http_parser.pyx @@ -0,0 +1,837 @@ +#cython: language_level=3 +# +# Based on https://github.com/MagicStack/httptools +# + +from cpython cimport ( + Py_buffer, + PyBUF_SIMPLE, + PyBuffer_Release, + PyBytes_AsString, + PyBytes_AsStringAndSize, + PyObject_GetBuffer, +) +from cpython.mem cimport PyMem_Free, PyMem_Malloc +from libc.limits cimport ULLONG_MAX +from libc.string cimport memcpy + +from multidict import CIMultiDict as _CIMultiDict, CIMultiDictProxy as _CIMultiDictProxy +from yarl import URL as _URL + +from aiohttp import hdrs +from aiohttp.helpers import DEBUG, set_exception + +from .http_exceptions import ( + BadHttpMessage, + BadHttpMethod, + BadStatusLine, + ContentLengthError, + InvalidHeader, + InvalidURLError, + LineTooLong, + PayloadEncodingError, + TransferEncodingError, +) +from .http_parser import DeflateBuffer as _DeflateBuffer +from .http_writer import ( + HttpVersion as _HttpVersion, + HttpVersion10 as _HttpVersion10, + HttpVersion11 as _HttpVersion11, +) +from .streams import EMPTY_PAYLOAD as _EMPTY_PAYLOAD, StreamReader as _StreamReader + +cimport cython + +from aiohttp cimport _cparser as cparser + +include "_headers.pxi" + +from aiohttp cimport _find_header + +ALLOWED_UPGRADES = frozenset({"websocket"}) +DEF DEFAULT_FREELIST_SIZE = 250 + +cdef extern from "Python.h": + int PyByteArray_Resize(object, Py_ssize_t) except -1 + Py_ssize_t PyByteArray_Size(object) except -1 + char* PyByteArray_AsString(object) + +__all__ = ('HttpRequestParser', 'HttpResponseParser', + 'RawRequestMessage', 'RawResponseMessage') + +cdef object URL = _URL +cdef object URL_build = URL.build +cdef object CIMultiDict = _CIMultiDict +cdef object CIMultiDictProxy = _CIMultiDictProxy +cdef object HttpVersion = _HttpVersion +cdef object HttpVersion10 = _HttpVersion10 +cdef object HttpVersion11 = _HttpVersion11 +cdef object SEC_WEBSOCKET_KEY1 = hdrs.SEC_WEBSOCKET_KEY1 +cdef object CONTENT_ENCODING = hdrs.CONTENT_ENCODING +cdef object EMPTY_PAYLOAD = _EMPTY_PAYLOAD +cdef object StreamReader = _StreamReader +cdef object DeflateBuffer = _DeflateBuffer +cdef bytes EMPTY_BYTES = b"" + +cdef inline object extend(object buf, const char* at, size_t length): + cdef Py_ssize_t s + cdef char* ptr + s = PyByteArray_Size(buf) + PyByteArray_Resize(buf, s + length) + ptr = PyByteArray_AsString(buf) + memcpy(ptr + s, at, length) + + +DEF METHODS_COUNT = 46; + +cdef list _http_method = [] + +for i in range(METHODS_COUNT): + _http_method.append( + cparser.llhttp_method_name( i).decode('ascii')) + + +cdef inline str http_method_str(int i): + if i < METHODS_COUNT: + return _http_method[i] + else: + return "" + +cdef inline object find_header(bytes raw_header): + cdef Py_ssize_t size + cdef char *buf + cdef int idx + PyBytes_AsStringAndSize(raw_header, &buf, &size) + idx = _find_header.find_header(buf, size) + if idx == -1: + return raw_header.decode('utf-8', 'surrogateescape') + return headers[idx] + + +@cython.freelist(DEFAULT_FREELIST_SIZE) +cdef class RawRequestMessage: + cdef readonly str method + cdef readonly str path + cdef readonly object version # HttpVersion + cdef readonly object headers # CIMultiDict + cdef readonly object raw_headers # tuple + cdef readonly object should_close + cdef readonly object compression + cdef readonly object upgrade + cdef readonly object chunked + cdef readonly object url # yarl.URL + + def __init__(self, method, path, version, headers, raw_headers, + should_close, compression, upgrade, chunked, url): + self.method = method + self.path = path + self.version = version + self.headers = headers + self.raw_headers = raw_headers + self.should_close = should_close + self.compression = compression + self.upgrade = upgrade + self.chunked = chunked + self.url = url + + def __repr__(self): + info = [] + info.append(("method", self.method)) + info.append(("path", self.path)) + info.append(("version", self.version)) + info.append(("headers", self.headers)) + info.append(("raw_headers", self.raw_headers)) + info.append(("should_close", self.should_close)) + info.append(("compression", self.compression)) + info.append(("upgrade", self.upgrade)) + info.append(("chunked", self.chunked)) + info.append(("url", self.url)) + sinfo = ', '.join(name + '=' + repr(val) for name, val in info) + return '' + + def _replace(self, **dct): + cdef RawRequestMessage ret + ret = _new_request_message(self.method, + self.path, + self.version, + self.headers, + self.raw_headers, + self.should_close, + self.compression, + self.upgrade, + self.chunked, + self.url) + if "method" in dct: + ret.method = dct["method"] + if "path" in dct: + ret.path = dct["path"] + if "version" in dct: + ret.version = dct["version"] + if "headers" in dct: + ret.headers = dct["headers"] + if "raw_headers" in dct: + ret.raw_headers = dct["raw_headers"] + if "should_close" in dct: + ret.should_close = dct["should_close"] + if "compression" in dct: + ret.compression = dct["compression"] + if "upgrade" in dct: + ret.upgrade = dct["upgrade"] + if "chunked" in dct: + ret.chunked = dct["chunked"] + if "url" in dct: + ret.url = dct["url"] + return ret + +cdef _new_request_message(str method, + str path, + object version, + object headers, + object raw_headers, + bint should_close, + object compression, + bint upgrade, + bint chunked, + object url): + cdef RawRequestMessage ret + ret = RawRequestMessage.__new__(RawRequestMessage) + ret.method = method + ret.path = path + ret.version = version + ret.headers = headers + ret.raw_headers = raw_headers + ret.should_close = should_close + ret.compression = compression + ret.upgrade = upgrade + ret.chunked = chunked + ret.url = url + return ret + + +@cython.freelist(DEFAULT_FREELIST_SIZE) +cdef class RawResponseMessage: + cdef readonly object version # HttpVersion + cdef readonly int code + cdef readonly str reason + cdef readonly object headers # CIMultiDict + cdef readonly object raw_headers # tuple + cdef readonly object should_close + cdef readonly object compression + cdef readonly object upgrade + cdef readonly object chunked + + def __init__(self, version, code, reason, headers, raw_headers, + should_close, compression, upgrade, chunked): + self.version = version + self.code = code + self.reason = reason + self.headers = headers + self.raw_headers = raw_headers + self.should_close = should_close + self.compression = compression + self.upgrade = upgrade + self.chunked = chunked + + def __repr__(self): + info = [] + info.append(("version", self.version)) + info.append(("code", self.code)) + info.append(("reason", self.reason)) + info.append(("headers", self.headers)) + info.append(("raw_headers", self.raw_headers)) + info.append(("should_close", self.should_close)) + info.append(("compression", self.compression)) + info.append(("upgrade", self.upgrade)) + info.append(("chunked", self.chunked)) + sinfo = ', '.join(name + '=' + repr(val) for name, val in info) + return '' + + +cdef _new_response_message(object version, + int code, + str reason, + object headers, + object raw_headers, + bint should_close, + object compression, + bint upgrade, + bint chunked): + cdef RawResponseMessage ret + ret = RawResponseMessage.__new__(RawResponseMessage) + ret.version = version + ret.code = code + ret.reason = reason + ret.headers = headers + ret.raw_headers = raw_headers + ret.should_close = should_close + ret.compression = compression + ret.upgrade = upgrade + ret.chunked = chunked + return ret + + +@cython.internal +cdef class HttpParser: + + cdef: + cparser.llhttp_t* _cparser + cparser.llhttp_settings_t* _csettings + + bytes _raw_name + object _name + bytes _raw_value + bint _has_value + + object _protocol + object _loop + object _timer + + size_t _max_line_size + size_t _max_field_size + size_t _max_headers + bint _response_with_body + bint _read_until_eof + + bint _started + object _url + bytearray _buf + str _path + str _reason + list _headers + list _raw_headers + bint _upgraded + list _messages + object _payload + bint _payload_error + object _payload_exception + object _last_error + bint _auto_decompress + int _limit + + str _content_encoding + + Py_buffer py_buf + + def __cinit__(self): + self._cparser = \ + PyMem_Malloc(sizeof(cparser.llhttp_t)) + if self._cparser is NULL: + raise MemoryError() + + self._csettings = \ + PyMem_Malloc(sizeof(cparser.llhttp_settings_t)) + if self._csettings is NULL: + raise MemoryError() + + def __dealloc__(self): + PyMem_Free(self._cparser) + PyMem_Free(self._csettings) + + cdef _init( + self, cparser.llhttp_type mode, + object protocol, object loop, int limit, + object timer=None, + size_t max_line_size=8190, size_t max_headers=32768, + size_t max_field_size=8190, payload_exception=None, + bint response_with_body=True, bint read_until_eof=False, + bint auto_decompress=True, + ): + cparser.llhttp_settings_init(self._csettings) + cparser.llhttp_init(self._cparser, mode, self._csettings) + self._cparser.data = self + self._cparser.content_length = 0 + + self._protocol = protocol + self._loop = loop + self._timer = timer + + self._buf = bytearray() + self._payload = None + self._payload_error = 0 + self._payload_exception = payload_exception + self._messages = [] + + self._raw_name = EMPTY_BYTES + self._raw_value = EMPTY_BYTES + self._has_value = False + + self._max_line_size = max_line_size + self._max_headers = max_headers + self._max_field_size = max_field_size + self._response_with_body = response_with_body + self._read_until_eof = read_until_eof + self._upgraded = False + self._auto_decompress = auto_decompress + self._content_encoding = None + + self._csettings.on_url = cb_on_url + self._csettings.on_status = cb_on_status + self._csettings.on_header_field = cb_on_header_field + self._csettings.on_header_value = cb_on_header_value + self._csettings.on_headers_complete = cb_on_headers_complete + self._csettings.on_body = cb_on_body + self._csettings.on_message_begin = cb_on_message_begin + self._csettings.on_message_complete = cb_on_message_complete + self._csettings.on_chunk_header = cb_on_chunk_header + self._csettings.on_chunk_complete = cb_on_chunk_complete + + self._last_error = None + self._limit = limit + + cdef _process_header(self): + cdef str value + if self._raw_name is not EMPTY_BYTES: + name = find_header(self._raw_name) + value = self._raw_value.decode('utf-8', 'surrogateescape') + + self._headers.append((name, value)) + + if name is CONTENT_ENCODING: + self._content_encoding = value + + self._has_value = False + self._raw_headers.append((self._raw_name, self._raw_value)) + self._raw_name = EMPTY_BYTES + self._raw_value = EMPTY_BYTES + + cdef _on_header_field(self, char* at, size_t length): + if self._has_value: + self._process_header() + + if self._raw_name is EMPTY_BYTES: + self._raw_name = at[:length] + else: + self._raw_name += at[:length] + + cdef _on_header_value(self, char* at, size_t length): + if self._raw_value is EMPTY_BYTES: + self._raw_value = at[:length] + else: + self._raw_value += at[:length] + self._has_value = True + + cdef _on_headers_complete(self): + self._process_header() + + should_close = not cparser.llhttp_should_keep_alive(self._cparser) + upgrade = self._cparser.upgrade + chunked = self._cparser.flags & cparser.F_CHUNKED + + raw_headers = tuple(self._raw_headers) + headers = CIMultiDictProxy(CIMultiDict(self._headers)) + + if self._cparser.type == cparser.HTTP_REQUEST: + allowed = upgrade and headers.get("upgrade", "").lower() in ALLOWED_UPGRADES + if allowed or self._cparser.method == cparser.HTTP_CONNECT: + self._upgraded = True + else: + if upgrade and self._cparser.status_code == 101: + self._upgraded = True + + # do not support old websocket spec + if SEC_WEBSOCKET_KEY1 in headers: + raise InvalidHeader(SEC_WEBSOCKET_KEY1) + + encoding = None + enc = self._content_encoding + if enc is not None: + self._content_encoding = None + enc = enc.lower() + if enc in ('gzip', 'deflate', 'br'): + encoding = enc + + if self._cparser.type == cparser.HTTP_REQUEST: + method = http_method_str(self._cparser.method) + msg = _new_request_message( + method, self._path, + self.http_version(), headers, raw_headers, + should_close, encoding, upgrade, chunked, self._url) + else: + msg = _new_response_message( + self.http_version(), self._cparser.status_code, self._reason, + headers, raw_headers, should_close, encoding, + upgrade, chunked) + + if ( + ULLONG_MAX > self._cparser.content_length > 0 or chunked or + self._cparser.method == cparser.HTTP_CONNECT or + (self._cparser.status_code >= 199 and + self._cparser.content_length == 0 and + self._read_until_eof) + ): + payload = StreamReader( + self._protocol, timer=self._timer, loop=self._loop, + limit=self._limit) + else: + payload = EMPTY_PAYLOAD + + self._payload = payload + if encoding is not None and self._auto_decompress: + self._payload = DeflateBuffer(payload, encoding) + + if not self._response_with_body: + payload = EMPTY_PAYLOAD + + self._messages.append((msg, payload)) + + cdef _on_message_complete(self): + self._payload.feed_eof() + self._payload = None + + cdef _on_chunk_header(self): + self._payload.begin_http_chunk_receiving() + + cdef _on_chunk_complete(self): + self._payload.end_http_chunk_receiving() + + cdef object _on_status_complete(self): + pass + + cdef inline http_version(self): + cdef cparser.llhttp_t* parser = self._cparser + + if parser.http_major == 1: + if parser.http_minor == 0: + return HttpVersion10 + elif parser.http_minor == 1: + return HttpVersion11 + + return HttpVersion(parser.http_major, parser.http_minor) + + ### Public API ### + + def feed_eof(self): + cdef bytes desc + + if self._payload is not None: + if self._cparser.flags & cparser.F_CHUNKED: + raise TransferEncodingError( + "Not enough data to satisfy transfer length header.") + elif self._cparser.flags & cparser.F_CONTENT_LENGTH: + raise ContentLengthError( + "Not enough data to satisfy content length header.") + elif cparser.llhttp_get_errno(self._cparser) != cparser.HPE_OK: + desc = cparser.llhttp_get_error_reason(self._cparser) + raise PayloadEncodingError(desc.decode('latin-1')) + else: + self._payload.feed_eof() + elif self._started: + self._on_headers_complete() + if self._messages: + return self._messages[-1][0] + + def feed_data(self, data): + cdef: + size_t data_len + size_t nb + cdef cparser.llhttp_errno_t errno + + PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) + data_len = self.py_buf.len + + errno = cparser.llhttp_execute( + self._cparser, + self.py_buf.buf, + data_len) + + if errno is cparser.HPE_PAUSED_UPGRADE: + cparser.llhttp_resume_after_upgrade(self._cparser) + + nb = cparser.llhttp_get_error_pos(self._cparser) - self.py_buf.buf + + PyBuffer_Release(&self.py_buf) + + if errno not in (cparser.HPE_OK, cparser.HPE_PAUSED_UPGRADE): + if self._payload_error == 0: + if self._last_error is not None: + ex = self._last_error + self._last_error = None + else: + after = cparser.llhttp_get_error_pos(self._cparser) + before = data[:after - self.py_buf.buf] + after_b = after.split(b"\r\n", 1)[0] + before = before.rsplit(b"\r\n", 1)[-1] + data = before + after_b + pointer = " " * (len(repr(before))-1) + "^" + ex = parser_error_from_errno(self._cparser, data, pointer) + self._payload = None + raise ex + + if self._messages: + messages = self._messages + self._messages = [] + else: + messages = () + + if self._upgraded: + return messages, True, data[nb:] + else: + return messages, False, b"" + + def set_upgraded(self, val): + self._upgraded = val + + +cdef class HttpRequestParser(HttpParser): + + def __init__( + self, protocol, loop, int limit, timer=None, + size_t max_line_size=8190, size_t max_headers=32768, + size_t max_field_size=8190, payload_exception=None, + bint response_with_body=True, bint read_until_eof=False, + bint auto_decompress=True, + ): + self._init(cparser.HTTP_REQUEST, protocol, loop, limit, timer, + max_line_size, max_headers, max_field_size, + payload_exception, response_with_body, read_until_eof, + auto_decompress) + + cdef object _on_status_complete(self): + cdef int idx1, idx2 + if not self._buf: + return + self._path = self._buf.decode('utf-8', 'surrogateescape') + try: + idx3 = len(self._path) + if self._cparser.method == cparser.HTTP_CONNECT: + # authority-form, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3 + self._url = URL.build(authority=self._path, encoded=True) + elif idx3 > 1 and self._path[0] == '/': + # origin-form, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.1 + idx1 = self._path.find("?") + if idx1 == -1: + query = "" + idx2 = self._path.find("#") + if idx2 == -1: + path = self._path + fragment = "" + else: + path = self._path[0: idx2] + fragment = self._path[idx2+1:] + + else: + path = self._path[0:idx1] + idx1 += 1 + idx2 = self._path.find("#", idx1+1) + if idx2 == -1: + query = self._path[idx1:] + fragment = "" + else: + query = self._path[idx1: idx2] + fragment = self._path[idx2+1:] + + self._url = URL.build( + path=path, + query_string=query, + fragment=fragment, + encoded=True, + ) + else: + # absolute-form for proxy maybe, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2 + self._url = URL(self._path, encoded=True) + finally: + PyByteArray_Resize(self._buf, 0) + + +cdef class HttpResponseParser(HttpParser): + + def __init__( + self, protocol, loop, int limit, timer=None, + size_t max_line_size=8190, size_t max_headers=32768, + size_t max_field_size=8190, payload_exception=None, + bint response_with_body=True, bint read_until_eof=False, + bint auto_decompress=True + ): + self._init(cparser.HTTP_RESPONSE, protocol, loop, limit, timer, + max_line_size, max_headers, max_field_size, + payload_exception, response_with_body, read_until_eof, + auto_decompress) + # Use strict parsing on dev mode, so users are warned about broken servers. + if not DEBUG: + cparser.llhttp_set_lenient_headers(self._cparser, 1) + cparser.llhttp_set_lenient_optional_cr_before_lf(self._cparser, 1) + cparser.llhttp_set_lenient_spaces_after_chunk_size(self._cparser, 1) + + cdef object _on_status_complete(self): + if self._buf: + self._reason = self._buf.decode('utf-8', 'surrogateescape') + PyByteArray_Resize(self._buf, 0) + else: + self._reason = self._reason or '' + +cdef int cb_on_message_begin(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + + pyparser._started = True + pyparser._headers = [] + pyparser._raw_headers = [] + PyByteArray_Resize(pyparser._buf, 0) + pyparser._path = None + pyparser._reason = None + return 0 + + +cdef int cb_on_url(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + try: + if length > pyparser._max_line_size: + raise LineTooLong( + 'Status line is too long', pyparser._max_line_size, length) + extend(pyparser._buf, at, length) + except BaseException as ex: + pyparser._last_error = ex + return -1 + else: + return 0 + + +cdef int cb_on_status(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + cdef str reason + try: + if length > pyparser._max_line_size: + raise LineTooLong( + 'Status line is too long', pyparser._max_line_size, length) + extend(pyparser._buf, at, length) + except BaseException as ex: + pyparser._last_error = ex + return -1 + else: + return 0 + + +cdef int cb_on_header_field(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + cdef Py_ssize_t size + try: + pyparser._on_status_complete() + size = len(pyparser._raw_name) + length + if size > pyparser._max_field_size: + raise LineTooLong( + 'Header name is too long', pyparser._max_field_size, size) + pyparser._on_header_field(at, length) + except BaseException as ex: + pyparser._last_error = ex + return -1 + else: + return 0 + + +cdef int cb_on_header_value(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + cdef Py_ssize_t size + try: + size = len(pyparser._raw_value) + length + if size > pyparser._max_field_size: + raise LineTooLong( + 'Header value is too long', pyparser._max_field_size, size) + pyparser._on_header_value(at, length) + except BaseException as ex: + pyparser._last_error = ex + return -1 + else: + return 0 + + +cdef int cb_on_headers_complete(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + try: + pyparser._on_status_complete() + pyparser._on_headers_complete() + except BaseException as exc: + pyparser._last_error = exc + return -1 + else: + if pyparser._upgraded or pyparser._cparser.method == cparser.HTTP_CONNECT: + return 2 + else: + return 0 + + +cdef int cb_on_body(cparser.llhttp_t* parser, + const char *at, size_t length) except -1: + cdef HttpParser pyparser = parser.data + cdef bytes body = at[:length] + try: + pyparser._payload.feed_data(body, length) + except BaseException as underlying_exc: + reraised_exc = underlying_exc + if pyparser._payload_exception is not None: + reraised_exc = pyparser._payload_exception(str(underlying_exc)) + + set_exception(pyparser._payload, reraised_exc, underlying_exc) + + pyparser._payload_error = 1 + return -1 + else: + return 0 + + +cdef int cb_on_message_complete(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + try: + pyparser._started = False + pyparser._on_message_complete() + except BaseException as exc: + pyparser._last_error = exc + return -1 + else: + return 0 + + +cdef int cb_on_chunk_header(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + try: + pyparser._on_chunk_header() + except BaseException as exc: + pyparser._last_error = exc + return -1 + else: + return 0 + + +cdef int cb_on_chunk_complete(cparser.llhttp_t* parser) except -1: + cdef HttpParser pyparser = parser.data + try: + pyparser._on_chunk_complete() + except BaseException as exc: + pyparser._last_error = exc + return -1 + else: + return 0 + + +cdef parser_error_from_errno(cparser.llhttp_t* parser, data, pointer): + cdef cparser.llhttp_errno_t errno = cparser.llhttp_get_errno(parser) + cdef bytes desc = cparser.llhttp_get_error_reason(parser) + + err_msg = "{}:\n\n {!r}\n {}".format(desc.decode("latin-1"), data, pointer) + + if errno in {cparser.HPE_CB_MESSAGE_BEGIN, + cparser.HPE_CB_HEADERS_COMPLETE, + cparser.HPE_CB_MESSAGE_COMPLETE, + cparser.HPE_CB_CHUNK_HEADER, + cparser.HPE_CB_CHUNK_COMPLETE, + cparser.HPE_INVALID_CONSTANT, + cparser.HPE_INVALID_HEADER_TOKEN, + cparser.HPE_INVALID_CONTENT_LENGTH, + cparser.HPE_INVALID_CHUNK_SIZE, + cparser.HPE_INVALID_EOF_STATE, + cparser.HPE_INVALID_TRANSFER_ENCODING}: + return BadHttpMessage(err_msg) + elif errno == cparser.HPE_INVALID_METHOD: + return BadHttpMethod(error=err_msg) + elif errno in {cparser.HPE_INVALID_STATUS, + cparser.HPE_INVALID_VERSION}: + return BadStatusLine(error=err_msg) + elif errno == cparser.HPE_INVALID_URL: + return InvalidURLError(err_msg) + + return BadHttpMessage(err_msg) diff --git a/lib/python3.1/site-packages/aiohttp/_http_writer.cpython-310-x86_64-linux-gnu.so b/lib/python3.1/site-packages/aiohttp/_http_writer.cpython-310-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..7208ef7d5bc2cc5b23aac4e81f9868a3959aee36 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/_http_writer.cpython-310-x86_64-linux-gnu.so differ diff --git a/lib/python3.1/site-packages/aiohttp/_http_writer.pyx b/lib/python3.1/site-packages/aiohttp/_http_writer.pyx new file mode 100644 index 0000000000000000000000000000000000000000..4a3ae1f9e682f7632e0234f2bf7e9a71823caca4 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_http_writer.pyx @@ -0,0 +1,160 @@ +from cpython.bytes cimport PyBytes_FromStringAndSize +from cpython.exc cimport PyErr_NoMemory +from cpython.mem cimport PyMem_Free, PyMem_Malloc, PyMem_Realloc +from cpython.object cimport PyObject_Str +from libc.stdint cimport uint8_t, uint64_t +from libc.string cimport memcpy + +from multidict import istr + +DEF BUF_SIZE = 16 * 1024 # 16KiB +cdef char BUFFER[BUF_SIZE] + +cdef object _istr = istr + + +# ----------------- writer --------------------------- + +cdef struct Writer: + char *buf + Py_ssize_t size + Py_ssize_t pos + + +cdef inline void _init_writer(Writer* writer): + writer.buf = &BUFFER[0] + writer.size = BUF_SIZE + writer.pos = 0 + + +cdef inline void _release_writer(Writer* writer): + if writer.buf != BUFFER: + PyMem_Free(writer.buf) + + +cdef inline int _write_byte(Writer* writer, uint8_t ch): + cdef char * buf + cdef Py_ssize_t size + + if writer.pos == writer.size: + # reallocate + size = writer.size + BUF_SIZE + if writer.buf == BUFFER: + buf = PyMem_Malloc(size) + if buf == NULL: + PyErr_NoMemory() + return -1 + memcpy(buf, writer.buf, writer.size) + else: + buf = PyMem_Realloc(writer.buf, size) + if buf == NULL: + PyErr_NoMemory() + return -1 + writer.buf = buf + writer.size = size + writer.buf[writer.pos] = ch + writer.pos += 1 + return 0 + + +cdef inline int _write_utf8(Writer* writer, Py_UCS4 symbol): + cdef uint64_t utf = symbol + + if utf < 0x80: + return _write_byte(writer, utf) + elif utf < 0x800: + if _write_byte(writer, (0xc0 | (utf >> 6))) < 0: + return -1 + return _write_byte(writer, (0x80 | (utf & 0x3f))) + elif 0xD800 <= utf <= 0xDFFF: + # surogate pair, ignored + return 0 + elif utf < 0x10000: + if _write_byte(writer, (0xe0 | (utf >> 12))) < 0: + return -1 + if _write_byte(writer, (0x80 | ((utf >> 6) & 0x3f))) < 0: + return -1 + return _write_byte(writer, (0x80 | (utf & 0x3f))) + elif utf > 0x10FFFF: + # symbol is too large + return 0 + else: + if _write_byte(writer, (0xf0 | (utf >> 18))) < 0: + return -1 + if _write_byte(writer, + (0x80 | ((utf >> 12) & 0x3f))) < 0: + return -1 + if _write_byte(writer, + (0x80 | ((utf >> 6) & 0x3f))) < 0: + return -1 + return _write_byte(writer, (0x80 | (utf & 0x3f))) + + +cdef inline int _write_str(Writer* writer, str s): + cdef Py_UCS4 ch + for ch in s: + if _write_utf8(writer, ch) < 0: + return -1 + + +cdef inline int _write_str_raise_on_nlcr(Writer* writer, object s): + cdef Py_UCS4 ch + cdef str out_str + if type(s) is str: + out_str = s + elif type(s) is _istr: + out_str = PyObject_Str(s) + elif not isinstance(s, str): + raise TypeError("Cannot serialize non-str key {!r}".format(s)) + else: + out_str = str(s) + + for ch in out_str: + if ch == 0x0D or ch == 0x0A: + raise ValueError( + "Newline or carriage return detected in headers. " + "Potential header injection attack." + ) + if _write_utf8(writer, ch) < 0: + return -1 + + +# --------------- _serialize_headers ---------------------- + +def _serialize_headers(str status_line, headers): + cdef Writer writer + cdef object key + cdef object val + + _init_writer(&writer) + + try: + if _write_str(&writer, status_line) < 0: + raise + if _write_byte(&writer, b'\r') < 0: + raise + if _write_byte(&writer, b'\n') < 0: + raise + + for key, val in headers.items(): + if _write_str_raise_on_nlcr(&writer, key) < 0: + raise + if _write_byte(&writer, b':') < 0: + raise + if _write_byte(&writer, b' ') < 0: + raise + if _write_str_raise_on_nlcr(&writer, val) < 0: + raise + if _write_byte(&writer, b'\r') < 0: + raise + if _write_byte(&writer, b'\n') < 0: + raise + + if _write_byte(&writer, b'\r') < 0: + raise + if _write_byte(&writer, b'\n') < 0: + raise + + return PyBytes_FromStringAndSize(writer.buf, writer.pos) + finally: + _release_writer(&writer) diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/.hash/mask.pxd.hash b/lib/python3.1/site-packages/aiohttp/_websocket/.hash/mask.pxd.hash new file mode 100644 index 0000000000000000000000000000000000000000..eadfed3d48b4781b271296d8864672a9b46b73c8 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_websocket/.hash/mask.pxd.hash @@ -0,0 +1 @@ +b01999d409b29bd916e067bc963d5f2d9ee63cfc9ae0bccb769910131417bf93 /home/runner/work/aiohttp/aiohttp/aiohttp/_websocket/mask.pxd diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/.hash/mask.pyx.hash b/lib/python3.1/site-packages/aiohttp/_websocket/.hash/mask.pyx.hash new file mode 100644 index 0000000000000000000000000000000000000000..5cd7ae67746cbdcc347183a9fdb1aa24a6f89063 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_websocket/.hash/mask.pyx.hash @@ -0,0 +1 @@ +0478ceb55d0ed30ef1a7da742cd003449bc69a07cf9fdb06789bd2b347cbfffe /home/runner/work/aiohttp/aiohttp/aiohttp/_websocket/mask.pyx diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/.hash/reader_c.pxd.hash b/lib/python3.1/site-packages/aiohttp/_websocket/.hash/reader_c.pxd.hash new file mode 100644 index 0000000000000000000000000000000000000000..f2feb6aad915eb2dfbe1803aec0f0095f67fb5d6 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_websocket/.hash/reader_c.pxd.hash @@ -0,0 +1 @@ +9e5fe78ed0ebce5414d2b8e01868d90c1facc20b84d2d5ff6c23e86e44a155ae /home/runner/work/aiohttp/aiohttp/aiohttp/_websocket/reader_c.pxd diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/__init__.py b/lib/python3.1/site-packages/aiohttp/_websocket/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..836257cc47aba3e74863c7de0e098d0835bcee1f --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_websocket/__init__.py @@ -0,0 +1 @@ +"""WebSocket protocol versions 13 and 8.""" diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b36e66b62e4510322125988b72ef10a3aea758f7 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/__init__.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/helpers.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/helpers.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f676895d09873f3c2bd6d0c5cb963e8462f42351 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/helpers.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/models.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/models.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc400537ac54cda831a76880ca40100ee852deb1 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/models.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/reader.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/reader.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f347fd4ae685f9df9cae8de99b7083ff6ac4295 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/reader.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/reader_c.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/reader_c.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3ce5208d027e97c3498235ce651e55b18753a08f Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/reader_c.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/reader_py.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/reader_py.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d96455f82322e318e3d0784554a6c0f54bf7ac7 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/reader_py.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/writer.cpython-310.pyc b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/writer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93528441104e07b2a1e36d0f5681f07e17aa39fe Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/_websocket/__pycache__/writer.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/helpers.py b/lib/python3.1/site-packages/aiohttp/_websocket/helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..0bb58df9228603fc1eb79a7a2cac8301217a36a1 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_websocket/helpers.py @@ -0,0 +1,147 @@ +"""Helpers for WebSocket protocol versions 13 and 8.""" + +import functools +import re +from struct import Struct +from typing import TYPE_CHECKING, Final, List, Optional, Pattern, Tuple + +from ..helpers import NO_EXTENSIONS +from .models import WSHandshakeError + +UNPACK_LEN3 = Struct("!Q").unpack_from +UNPACK_CLOSE_CODE = Struct("!H").unpack +PACK_LEN1 = Struct("!BB").pack +PACK_LEN2 = Struct("!BBH").pack +PACK_LEN3 = Struct("!BBQ").pack +PACK_CLOSE_CODE = Struct("!H").pack +PACK_RANDBITS = Struct("!L").pack +MSG_SIZE: Final[int] = 2**14 +MASK_LEN: Final[int] = 4 + +WS_KEY: Final[bytes] = b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + + +# Used by _websocket_mask_python +@functools.lru_cache +def _xor_table() -> List[bytes]: + return [bytes(a ^ b for a in range(256)) for b in range(256)] + + +def _websocket_mask_python(mask: bytes, data: bytearray) -> None: + """Websocket masking function. + + `mask` is a `bytes` object of length 4; `data` is a `bytearray` + object of any length. The contents of `data` are masked with `mask`, + as specified in section 5.3 of RFC 6455. + + Note that this function mutates the `data` argument. + + This pure-python implementation may be replaced by an optimized + version when available. + + """ + assert isinstance(data, bytearray), data + assert len(mask) == 4, mask + + if data: + _XOR_TABLE = _xor_table() + a, b, c, d = (_XOR_TABLE[n] for n in mask) + data[::4] = data[::4].translate(a) + data[1::4] = data[1::4].translate(b) + data[2::4] = data[2::4].translate(c) + data[3::4] = data[3::4].translate(d) + + +if TYPE_CHECKING or NO_EXTENSIONS: # pragma: no cover + websocket_mask = _websocket_mask_python +else: + try: + from .mask import _websocket_mask_cython # type: ignore[import-not-found] + + websocket_mask = _websocket_mask_cython + except ImportError: # pragma: no cover + websocket_mask = _websocket_mask_python + + +_WS_EXT_RE: Final[Pattern[str]] = re.compile( + r"^(?:;\s*(?:" + r"(server_no_context_takeover)|" + r"(client_no_context_takeover)|" + r"(server_max_window_bits(?:=(\d+))?)|" + r"(client_max_window_bits(?:=(\d+))?)))*$" +) + +_WS_EXT_RE_SPLIT: Final[Pattern[str]] = re.compile(r"permessage-deflate([^,]+)?") + + +def ws_ext_parse(extstr: Optional[str], isserver: bool = False) -> Tuple[int, bool]: + if not extstr: + return 0, False + + compress = 0 + notakeover = False + for ext in _WS_EXT_RE_SPLIT.finditer(extstr): + defext = ext.group(1) + # Return compress = 15 when get `permessage-deflate` + if not defext: + compress = 15 + break + match = _WS_EXT_RE.match(defext) + if match: + compress = 15 + if isserver: + # Server never fail to detect compress handshake. + # Server does not need to send max wbit to client + if match.group(4): + compress = int(match.group(4)) + # Group3 must match if group4 matches + # Compress wbit 8 does not support in zlib + # If compress level not support, + # CONTINUE to next extension + if compress > 15 or compress < 9: + compress = 0 + continue + if match.group(1): + notakeover = True + # Ignore regex group 5 & 6 for client_max_window_bits + break + else: + if match.group(6): + compress = int(match.group(6)) + # Group5 must match if group6 matches + # Compress wbit 8 does not support in zlib + # If compress level not support, + # FAIL the parse progress + if compress > 15 or compress < 9: + raise WSHandshakeError("Invalid window size") + if match.group(2): + notakeover = True + # Ignore regex group 5 & 6 for client_max_window_bits + break + # Return Fail if client side and not match + elif not isserver: + raise WSHandshakeError("Extension for deflate not supported" + ext.group(1)) + + return compress, notakeover + + +def ws_ext_gen( + compress: int = 15, isserver: bool = False, server_notakeover: bool = False +) -> str: + # client_notakeover=False not used for server + # compress wbit 8 does not support in zlib + if compress < 9 or compress > 15: + raise ValueError( + "Compress wbits must between 9 and 15, zlib does not support wbits=8" + ) + enabledext = ["permessage-deflate"] + if not isserver: + enabledext.append("client_max_window_bits") + + if compress < 15: + enabledext.append("server_max_window_bits=" + str(compress)) + if server_notakeover: + enabledext.append("server_no_context_takeover") + # if client_notakeover: + # enabledext.append('client_no_context_takeover') + return "; ".join(enabledext) diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/mask.cpython-310-x86_64-linux-gnu.so b/lib/python3.1/site-packages/aiohttp/_websocket/mask.cpython-310-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..bfb080786ea6acd10e3561ba21472f79eae61387 Binary files /dev/null and b/lib/python3.1/site-packages/aiohttp/_websocket/mask.cpython-310-x86_64-linux-gnu.so differ diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/mask.pxd b/lib/python3.1/site-packages/aiohttp/_websocket/mask.pxd new file mode 100644 index 0000000000000000000000000000000000000000..90983de9ac7e59dfceb639c1e7b656abd5fbb305 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_websocket/mask.pxd @@ -0,0 +1,3 @@ +"""Cython declarations for websocket masking.""" + +cpdef void _websocket_mask_cython(bytes mask, bytearray data) diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/mask.pyx b/lib/python3.1/site-packages/aiohttp/_websocket/mask.pyx new file mode 100644 index 0000000000000000000000000000000000000000..2d956c8899644d4c6bce042b928be1f23e51293a --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_websocket/mask.pyx @@ -0,0 +1,48 @@ +from cpython cimport PyBytes_AsString + + +#from cpython cimport PyByteArray_AsString # cython still not exports that +cdef extern from "Python.h": + char* PyByteArray_AsString(bytearray ba) except NULL + +from libc.stdint cimport uint32_t, uint64_t, uintmax_t + + +cpdef void _websocket_mask_cython(bytes mask, bytearray data): + """Note, this function mutates its `data` argument + """ + cdef: + Py_ssize_t data_len, i + # bit operations on signed integers are implementation-specific + unsigned char * in_buf + const unsigned char * mask_buf + uint32_t uint32_msk + uint64_t uint64_msk + + assert len(mask) == 4 + + data_len = len(data) + in_buf = PyByteArray_AsString(data) + mask_buf = PyBytes_AsString(mask) + uint32_msk = (mask_buf)[0] + + # TODO: align in_data ptr to achieve even faster speeds + # does it need in python ?! malloc() always aligns to sizeof(long) bytes + + if sizeof(size_t) >= 8: + uint64_msk = uint32_msk + uint64_msk = (uint64_msk << 32) | uint32_msk + + while data_len >= 8: + (in_buf)[0] ^= uint64_msk + in_buf += 8 + data_len -= 8 + + + while data_len >= 4: + (in_buf)[0] ^= uint32_msk + in_buf += 4 + data_len -= 4 + + for i in range(0, data_len): + in_buf[i] ^= mask_buf[i] diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/models.py b/lib/python3.1/site-packages/aiohttp/_websocket/models.py new file mode 100644 index 0000000000000000000000000000000000000000..7e89b9652957e8f4e73916e18048368e8d75911e --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_websocket/models.py @@ -0,0 +1,84 @@ +"""Models for WebSocket protocol versions 13 and 8.""" + +import json +from enum import IntEnum +from typing import Any, Callable, Final, NamedTuple, Optional, cast + +WS_DEFLATE_TRAILING: Final[bytes] = bytes([0x00, 0x00, 0xFF, 0xFF]) + + +class WSCloseCode(IntEnum): + OK = 1000 + GOING_AWAY = 1001 + PROTOCOL_ERROR = 1002 + UNSUPPORTED_DATA = 1003 + ABNORMAL_CLOSURE = 1006 + INVALID_TEXT = 1007 + POLICY_VIOLATION = 1008 + MESSAGE_TOO_BIG = 1009 + MANDATORY_EXTENSION = 1010 + INTERNAL_ERROR = 1011 + SERVICE_RESTART = 1012 + TRY_AGAIN_LATER = 1013 + BAD_GATEWAY = 1014 + + +class WSMsgType(IntEnum): + # websocket spec types + CONTINUATION = 0x0 + TEXT = 0x1 + BINARY = 0x2 + PING = 0x9 + PONG = 0xA + CLOSE = 0x8 + + # aiohttp specific types + CLOSING = 0x100 + CLOSED = 0x101 + ERROR = 0x102 + + text = TEXT + binary = BINARY + ping = PING + pong = PONG + close = CLOSE + closing = CLOSING + closed = CLOSED + error = ERROR + + +class WSMessage(NamedTuple): + type: WSMsgType + # To type correctly, this would need some kind of tagged union for each type. + data: Any + extra: Optional[str] + + def json(self, *, loads: Callable[[Any], Any] = json.loads) -> Any: + """Return parsed JSON data. + + .. versionadded:: 0.22 + """ + return loads(self.data) + + +# Constructing the tuple directly to avoid the overhead of +# the lambda and arg processing since NamedTuples are constructed +# with a run time built lambda +# https://github.com/python/cpython/blob/d83fcf8371f2f33c7797bc8f5423a8bca8c46e5c/Lib/collections/__init__.py#L441 +WS_CLOSED_MESSAGE = tuple.__new__(WSMessage, (WSMsgType.CLOSED, None, None)) +WS_CLOSING_MESSAGE = tuple.__new__(WSMessage, (WSMsgType.CLOSING, None, None)) + + +class WebSocketError(Exception): + """WebSocket protocol parser error.""" + + def __init__(self, code: int, message: str) -> None: + self.code = code + super().__init__(code, message) + + def __str__(self) -> str: + return cast(str, self.args[1]) + + +class WSHandshakeError(Exception): + """WebSocket protocol handshake error.""" diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/reader.py b/lib/python3.1/site-packages/aiohttp/_websocket/reader.py new file mode 100644 index 0000000000000000000000000000000000000000..23f32265cfccbdc8c8fe2f1600accbfb6f816efa --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_websocket/reader.py @@ -0,0 +1,31 @@ +"""Reader for WebSocket protocol versions 13 and 8.""" + +from typing import TYPE_CHECKING + +from ..helpers import NO_EXTENSIONS + +if TYPE_CHECKING or NO_EXTENSIONS: # pragma: no cover + from .reader_py import ( + WebSocketDataQueue as WebSocketDataQueuePython, + WebSocketReader as WebSocketReaderPython, + ) + + WebSocketReader = WebSocketReaderPython + WebSocketDataQueue = WebSocketDataQueuePython +else: + try: + from .reader_c import ( # type: ignore[import-not-found] + WebSocketDataQueue as WebSocketDataQueueCython, + WebSocketReader as WebSocketReaderCython, + ) + + WebSocketReader = WebSocketReaderCython + WebSocketDataQueue = WebSocketDataQueueCython + except ImportError: # pragma: no cover + from .reader_py import ( + WebSocketDataQueue as WebSocketDataQueuePython, + WebSocketReader as WebSocketReaderPython, + ) + + WebSocketReader = WebSocketReaderPython + WebSocketDataQueue = WebSocketDataQueuePython diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/reader_c.pxd b/lib/python3.1/site-packages/aiohttp/_websocket/reader_c.pxd new file mode 100644 index 0000000000000000000000000000000000000000..a7620d8e87fddd189ed9dbb062e6ec34aa8d673e --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_websocket/reader_c.pxd @@ -0,0 +1,110 @@ +import cython + +from .mask cimport _websocket_mask_cython as websocket_mask + + +cdef unsigned int READ_HEADER +cdef unsigned int READ_PAYLOAD_LENGTH +cdef unsigned int READ_PAYLOAD_MASK +cdef unsigned int READ_PAYLOAD + +cdef int OP_CODE_NOT_SET +cdef int OP_CODE_CONTINUATION +cdef int OP_CODE_TEXT +cdef int OP_CODE_BINARY +cdef int OP_CODE_CLOSE +cdef int OP_CODE_PING +cdef int OP_CODE_PONG + +cdef int COMPRESSED_NOT_SET +cdef int COMPRESSED_FALSE +cdef int COMPRESSED_TRUE + +cdef object UNPACK_LEN3 +cdef object UNPACK_CLOSE_CODE +cdef object TUPLE_NEW + +cdef object WSMsgType +cdef object WSMessage + +cdef object WS_MSG_TYPE_TEXT +cdef object WS_MSG_TYPE_BINARY + +cdef set ALLOWED_CLOSE_CODES +cdef set MESSAGE_TYPES_WITH_CONTENT + +cdef tuple EMPTY_FRAME +cdef tuple EMPTY_FRAME_ERROR + +cdef class WebSocketDataQueue: + + cdef unsigned int _size + cdef public object _protocol + cdef unsigned int _limit + cdef object _loop + cdef bint _eof + cdef object _waiter + cdef object _exception + cdef public object _buffer + cdef object _get_buffer + cdef object _put_buffer + + cdef void _release_waiter(self) + + cpdef void feed_data(self, object data, unsigned int size) + + @cython.locals(size="unsigned int") + cdef _read_from_buffer(self) + +cdef class WebSocketReader: + + cdef WebSocketDataQueue queue + cdef unsigned int _max_msg_size + + cdef Exception _exc + cdef bytearray _partial + cdef unsigned int _state + + cdef int _opcode + cdef bint _frame_fin + cdef int _frame_opcode + cdef list _payload_fragments + cdef Py_ssize_t _frame_payload_len + + cdef bytes _tail + cdef bint _has_mask + cdef bytes _frame_mask + cdef Py_ssize_t _payload_bytes_to_read + cdef unsigned int _payload_len_flag + cdef int _compressed + cdef object _decompressobj + cdef bint _compress + + cpdef tuple feed_data(self, object data) + + @cython.locals( + is_continuation=bint, + fin=bint, + has_partial=bint, + payload_merged=bytes, + ) + cpdef void _handle_frame(self, bint fin, int opcode, object payload, int compressed) except * + + @cython.locals( + start_pos=Py_ssize_t, + data_len=Py_ssize_t, + length=Py_ssize_t, + chunk_size=Py_ssize_t, + chunk_len=Py_ssize_t, + data_len=Py_ssize_t, + data_cstr="const unsigned char *", + first_byte="unsigned char", + second_byte="unsigned char", + f_start_pos=Py_ssize_t, + f_end_pos=Py_ssize_t, + has_mask=bint, + fin=bint, + had_fragments=Py_ssize_t, + payload_bytearray=bytearray, + ) + cpdef void _feed_data(self, bytes data) except * diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/reader_c.py b/lib/python3.1/site-packages/aiohttp/_websocket/reader_c.py new file mode 100644 index 0000000000000000000000000000000000000000..f966a1593c5bc534442d1bd1a1067b0998969b28 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_websocket/reader_c.py @@ -0,0 +1,476 @@ +"""Reader for WebSocket protocol versions 13 and 8.""" + +import asyncio +import builtins +from collections import deque +from typing import Deque, Final, Optional, Set, Tuple, Union + +from ..base_protocol import BaseProtocol +from ..compression_utils import ZLibDecompressor +from ..helpers import _EXC_SENTINEL, set_exception +from ..streams import EofStream +from .helpers import UNPACK_CLOSE_CODE, UNPACK_LEN3, websocket_mask +from .models import ( + WS_DEFLATE_TRAILING, + WebSocketError, + WSCloseCode, + WSMessage, + WSMsgType, +) + +ALLOWED_CLOSE_CODES: Final[Set[int]] = {int(i) for i in WSCloseCode} + +# States for the reader, used to parse the WebSocket frame +# integer values are used so they can be cythonized +READ_HEADER = 1 +READ_PAYLOAD_LENGTH = 2 +READ_PAYLOAD_MASK = 3 +READ_PAYLOAD = 4 + +WS_MSG_TYPE_BINARY = WSMsgType.BINARY +WS_MSG_TYPE_TEXT = WSMsgType.TEXT + +# WSMsgType values unpacked so they can by cythonized to ints +OP_CODE_NOT_SET = -1 +OP_CODE_CONTINUATION = WSMsgType.CONTINUATION.value +OP_CODE_TEXT = WSMsgType.TEXT.value +OP_CODE_BINARY = WSMsgType.BINARY.value +OP_CODE_CLOSE = WSMsgType.CLOSE.value +OP_CODE_PING = WSMsgType.PING.value +OP_CODE_PONG = WSMsgType.PONG.value + +EMPTY_FRAME_ERROR = (True, b"") +EMPTY_FRAME = (False, b"") + +COMPRESSED_NOT_SET = -1 +COMPRESSED_FALSE = 0 +COMPRESSED_TRUE = 1 + +TUPLE_NEW = tuple.__new__ + +cython_int = int # Typed to int in Python, but cython with use a signed int in the pxd + + +class WebSocketDataQueue: + """WebSocketDataQueue resumes and pauses an underlying stream. + + It is a destination for WebSocket data. + """ + + def __init__( + self, protocol: BaseProtocol, limit: int, *, loop: asyncio.AbstractEventLoop + ) -> None: + self._size = 0 + self._protocol = protocol + self._limit = limit * 2 + self._loop = loop + self._eof = False + self._waiter: Optional[asyncio.Future[None]] = None + self._exception: Union[BaseException, None] = None + self._buffer: Deque[Tuple[WSMessage, int]] = deque() + self._get_buffer = self._buffer.popleft + self._put_buffer = self._buffer.append + + def is_eof(self) -> bool: + return self._eof + + def exception(self) -> Optional[BaseException]: + return self._exception + + def set_exception( + self, + exc: BaseException, + exc_cause: builtins.BaseException = _EXC_SENTINEL, + ) -> None: + self._eof = True + self._exception = exc + if (waiter := self._waiter) is not None: + self._waiter = None + set_exception(waiter, exc, exc_cause) + + def _release_waiter(self) -> None: + if (waiter := self._waiter) is None: + return + self._waiter = None + if not waiter.done(): + waiter.set_result(None) + + def feed_eof(self) -> None: + self._eof = True + self._release_waiter() + self._exception = None # Break cyclic references + + def feed_data(self, data: "WSMessage", size: "cython_int") -> None: + self._size += size + self._put_buffer((data, size)) + self._release_waiter() + if self._size > self._limit and not self._protocol._reading_paused: + self._protocol.pause_reading() + + async def read(self) -> WSMessage: + if not self._buffer and not self._eof: + assert not self._waiter + self._waiter = self._loop.create_future() + try: + await self._waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._waiter = None + raise + return self._read_from_buffer() + + def _read_from_buffer(self) -> WSMessage: + if self._buffer: + data, size = self._get_buffer() + self._size -= size + if self._size < self._limit and self._protocol._reading_paused: + self._protocol.resume_reading() + return data + if self._exception is not None: + raise self._exception + raise EofStream + + +class WebSocketReader: + def __init__( + self, queue: WebSocketDataQueue, max_msg_size: int, compress: bool = True + ) -> None: + self.queue = queue + self._max_msg_size = max_msg_size + + self._exc: Optional[Exception] = None + self._partial = bytearray() + self._state = READ_HEADER + + self._opcode: int = OP_CODE_NOT_SET + self._frame_fin = False + self._frame_opcode: int = OP_CODE_NOT_SET + self._payload_fragments: list[bytes] = [] + self._frame_payload_len = 0 + + self._tail: bytes = b"" + self._has_mask = False + self._frame_mask: Optional[bytes] = None + self._payload_bytes_to_read = 0 + self._payload_len_flag = 0 + self._compressed: int = COMPRESSED_NOT_SET + self._decompressobj: Optional[ZLibDecompressor] = None + self._compress = compress + + def feed_eof(self) -> None: + self.queue.feed_eof() + + # data can be bytearray on Windows because proactor event loop uses bytearray + # and asyncio types this to Union[bytes, bytearray, memoryview] so we need + # coerce data to bytes if it is not + def feed_data( + self, data: Union[bytes, bytearray, memoryview] + ) -> Tuple[bool, bytes]: + if type(data) is not bytes: + data = bytes(data) + + if self._exc is not None: + return True, data + + try: + self._feed_data(data) + except Exception as exc: + self._exc = exc + set_exception(self.queue, exc) + return EMPTY_FRAME_ERROR + + return EMPTY_FRAME + + def _handle_frame( + self, + fin: bool, + opcode: Union[int, cython_int], # Union intended: Cython pxd uses C int + payload: Union[bytes, bytearray], + compressed: Union[int, cython_int], # Union intended: Cython pxd uses C int + ) -> None: + msg: WSMessage + if opcode in {OP_CODE_TEXT, OP_CODE_BINARY, OP_CODE_CONTINUATION}: + # load text/binary + if not fin: + # got partial frame payload + if opcode != OP_CODE_CONTINUATION: + self._opcode = opcode + self._partial += payload + if self._max_msg_size and len(self._partial) >= self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Message size {len(self._partial)} " + f"exceeds limit {self._max_msg_size}", + ) + return + + has_partial = bool(self._partial) + if opcode == OP_CODE_CONTINUATION: + if self._opcode == OP_CODE_NOT_SET: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Continuation frame for non started message", + ) + opcode = self._opcode + self._opcode = OP_CODE_NOT_SET + # previous frame was non finished + # we should get continuation opcode + elif has_partial: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "The opcode in non-fin frame is expected " + f"to be zero, got {opcode!r}", + ) + + assembled_payload: Union[bytes, bytearray] + if has_partial: + assembled_payload = self._partial + payload + self._partial.clear() + else: + assembled_payload = payload + + if self._max_msg_size and len(assembled_payload) >= self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Message size {len(assembled_payload)} " + f"exceeds limit {self._max_msg_size}", + ) + + # Decompress process must to be done after all packets + # received. + if compressed: + if not self._decompressobj: + self._decompressobj = ZLibDecompressor(suppress_deflate_header=True) + # XXX: It's possible that the zlib backend (isal is known to + # do this, maybe others too?) will return max_length bytes, + # but internally buffer more data such that the payload is + # >max_length, so we return one extra byte and if we're able + # to do that, then the message is too big. + payload_merged = self._decompressobj.decompress_sync( + assembled_payload + WS_DEFLATE_TRAILING, + ( + self._max_msg_size + 1 + if self._max_msg_size + else self._max_msg_size + ), + ) + if self._max_msg_size and len(payload_merged) > self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Decompressed message exceeds size limit {self._max_msg_size}", + ) + elif type(assembled_payload) is bytes: + payload_merged = assembled_payload + else: + payload_merged = bytes(assembled_payload) + + if opcode == OP_CODE_TEXT: + try: + text = payload_merged.decode("utf-8") + except UnicodeDecodeError as exc: + raise WebSocketError( + WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" + ) from exc + + # XXX: The Text and Binary messages here can be a performance + # bottleneck, so we use tuple.__new__ to improve performance. + # This is not type safe, but many tests should fail in + # test_client_ws_functional.py if this is wrong. + self.queue.feed_data( + TUPLE_NEW(WSMessage, (WS_MSG_TYPE_TEXT, text, "")), + len(payload_merged), + ) + else: + self.queue.feed_data( + TUPLE_NEW(WSMessage, (WS_MSG_TYPE_BINARY, payload_merged, "")), + len(payload_merged), + ) + elif opcode == OP_CODE_CLOSE: + if len(payload) >= 2: + close_code = UNPACK_CLOSE_CODE(payload[:2])[0] + if close_code < 3000 and close_code not in ALLOWED_CLOSE_CODES: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Invalid close code: {close_code}", + ) + try: + close_message = payload[2:].decode("utf-8") + except UnicodeDecodeError as exc: + raise WebSocketError( + WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" + ) from exc + msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, close_code, close_message)) + elif payload: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Invalid close frame: {fin} {opcode} {payload!r}", + ) + else: + msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, 0, "")) + + self.queue.feed_data(msg, 0) + elif opcode == OP_CODE_PING: + msg = TUPLE_NEW(WSMessage, (WSMsgType.PING, payload, "")) + self.queue.feed_data(msg, len(payload)) + elif opcode == OP_CODE_PONG: + msg = TUPLE_NEW(WSMessage, (WSMsgType.PONG, payload, "")) + self.queue.feed_data(msg, len(payload)) + else: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, f"Unexpected opcode={opcode!r}" + ) + + def _feed_data(self, data: bytes) -> None: + """Return the next frame from the socket.""" + if self._tail: + data, self._tail = self._tail + data, b"" + + start_pos: int = 0 + data_len = len(data) + data_cstr = data + + while True: + # read header + if self._state == READ_HEADER: + if data_len - start_pos < 2: + break + first_byte = data_cstr[start_pos] + second_byte = data_cstr[start_pos + 1] + start_pos += 2 + + fin = (first_byte >> 7) & 1 + rsv1 = (first_byte >> 6) & 1 + rsv2 = (first_byte >> 5) & 1 + rsv3 = (first_byte >> 4) & 1 + opcode = first_byte & 0xF + + # frame-fin = %x0 ; more frames of this message follow + # / %x1 ; final frame of this message + # frame-rsv1 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # frame-rsv2 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # frame-rsv3 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # + # Remove rsv1 from this test for deflate development + if rsv2 or rsv3 or (rsv1 and not self._compress): + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received frame with non-zero reserved bits", + ) + + if opcode > 0x7 and fin == 0: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received fragmented control frame", + ) + + has_mask = (second_byte >> 7) & 1 + length = second_byte & 0x7F + + # Control frames MUST have a payload + # length of 125 bytes or less + if opcode > 0x7 and length > 125: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Control frame payload cannot be larger than 125 bytes", + ) + + # Set compress status if last package is FIN + # OR set compress status if this is first fragment + # Raise error if not first fragment with rsv1 = 0x1 + if self._frame_fin or self._compressed == COMPRESSED_NOT_SET: + self._compressed = COMPRESSED_TRUE if rsv1 else COMPRESSED_FALSE + elif rsv1: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received frame with non-zero reserved bits", + ) + + self._frame_fin = bool(fin) + self._frame_opcode = opcode + self._has_mask = bool(has_mask) + self._payload_len_flag = length + self._state = READ_PAYLOAD_LENGTH + + # read payload length + if self._state == READ_PAYLOAD_LENGTH: + len_flag = self._payload_len_flag + if len_flag == 126: + if data_len - start_pos < 2: + break + first_byte = data_cstr[start_pos] + second_byte = data_cstr[start_pos + 1] + start_pos += 2 + self._payload_bytes_to_read = first_byte << 8 | second_byte + elif len_flag > 126: + if data_len - start_pos < 8: + break + self._payload_bytes_to_read = UNPACK_LEN3(data, start_pos)[0] + start_pos += 8 + else: + self._payload_bytes_to_read = len_flag + + self._state = READ_PAYLOAD_MASK if self._has_mask else READ_PAYLOAD + + # read payload mask + if self._state == READ_PAYLOAD_MASK: + if data_len - start_pos < 4: + break + self._frame_mask = data_cstr[start_pos : start_pos + 4] + start_pos += 4 + self._state = READ_PAYLOAD + + if self._state == READ_PAYLOAD: + chunk_len = data_len - start_pos + if self._payload_bytes_to_read >= chunk_len: + f_end_pos = data_len + self._payload_bytes_to_read -= chunk_len + else: + f_end_pos = start_pos + self._payload_bytes_to_read + self._payload_bytes_to_read = 0 + + had_fragments = self._frame_payload_len + self._frame_payload_len += f_end_pos - start_pos + f_start_pos = start_pos + start_pos = f_end_pos + + if self._payload_bytes_to_read != 0: + # If we don't have a complete frame, we need to save the + # data for the next call to feed_data. + self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) + break + + payload: Union[bytes, bytearray] + if had_fragments: + # We have to join the payload fragments get the payload + self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) + if self._has_mask: + assert self._frame_mask is not None + payload_bytearray = bytearray(b"".join(self._payload_fragments)) + websocket_mask(self._frame_mask, payload_bytearray) + payload = payload_bytearray + else: + payload = b"".join(self._payload_fragments) + self._payload_fragments.clear() + elif self._has_mask: + assert self._frame_mask is not None + payload_bytearray = data_cstr[f_start_pos:f_end_pos] # type: ignore[assignment] + if type(payload_bytearray) is not bytearray: # pragma: no branch + # Cython will do the conversion for us + # but we need to do it for Python and we + # will always get here in Python + payload_bytearray = bytearray(payload_bytearray) + websocket_mask(self._frame_mask, payload_bytearray) + payload = payload_bytearray + else: + payload = data_cstr[f_start_pos:f_end_pos] + + self._handle_frame( + self._frame_fin, self._frame_opcode, payload, self._compressed + ) + self._frame_payload_len = 0 + self._state = READ_HEADER + + # XXX: Cython needs slices to be bounded, so we can't omit the slice end here. + self._tail = data_cstr[start_pos:data_len] if start_pos < data_len else b"" diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/reader_py.py b/lib/python3.1/site-packages/aiohttp/_websocket/reader_py.py new file mode 100644 index 0000000000000000000000000000000000000000..f966a1593c5bc534442d1bd1a1067b0998969b28 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_websocket/reader_py.py @@ -0,0 +1,476 @@ +"""Reader for WebSocket protocol versions 13 and 8.""" + +import asyncio +import builtins +from collections import deque +from typing import Deque, Final, Optional, Set, Tuple, Union + +from ..base_protocol import BaseProtocol +from ..compression_utils import ZLibDecompressor +from ..helpers import _EXC_SENTINEL, set_exception +from ..streams import EofStream +from .helpers import UNPACK_CLOSE_CODE, UNPACK_LEN3, websocket_mask +from .models import ( + WS_DEFLATE_TRAILING, + WebSocketError, + WSCloseCode, + WSMessage, + WSMsgType, +) + +ALLOWED_CLOSE_CODES: Final[Set[int]] = {int(i) for i in WSCloseCode} + +# States for the reader, used to parse the WebSocket frame +# integer values are used so they can be cythonized +READ_HEADER = 1 +READ_PAYLOAD_LENGTH = 2 +READ_PAYLOAD_MASK = 3 +READ_PAYLOAD = 4 + +WS_MSG_TYPE_BINARY = WSMsgType.BINARY +WS_MSG_TYPE_TEXT = WSMsgType.TEXT + +# WSMsgType values unpacked so they can by cythonized to ints +OP_CODE_NOT_SET = -1 +OP_CODE_CONTINUATION = WSMsgType.CONTINUATION.value +OP_CODE_TEXT = WSMsgType.TEXT.value +OP_CODE_BINARY = WSMsgType.BINARY.value +OP_CODE_CLOSE = WSMsgType.CLOSE.value +OP_CODE_PING = WSMsgType.PING.value +OP_CODE_PONG = WSMsgType.PONG.value + +EMPTY_FRAME_ERROR = (True, b"") +EMPTY_FRAME = (False, b"") + +COMPRESSED_NOT_SET = -1 +COMPRESSED_FALSE = 0 +COMPRESSED_TRUE = 1 + +TUPLE_NEW = tuple.__new__ + +cython_int = int # Typed to int in Python, but cython with use a signed int in the pxd + + +class WebSocketDataQueue: + """WebSocketDataQueue resumes and pauses an underlying stream. + + It is a destination for WebSocket data. + """ + + def __init__( + self, protocol: BaseProtocol, limit: int, *, loop: asyncio.AbstractEventLoop + ) -> None: + self._size = 0 + self._protocol = protocol + self._limit = limit * 2 + self._loop = loop + self._eof = False + self._waiter: Optional[asyncio.Future[None]] = None + self._exception: Union[BaseException, None] = None + self._buffer: Deque[Tuple[WSMessage, int]] = deque() + self._get_buffer = self._buffer.popleft + self._put_buffer = self._buffer.append + + def is_eof(self) -> bool: + return self._eof + + def exception(self) -> Optional[BaseException]: + return self._exception + + def set_exception( + self, + exc: BaseException, + exc_cause: builtins.BaseException = _EXC_SENTINEL, + ) -> None: + self._eof = True + self._exception = exc + if (waiter := self._waiter) is not None: + self._waiter = None + set_exception(waiter, exc, exc_cause) + + def _release_waiter(self) -> None: + if (waiter := self._waiter) is None: + return + self._waiter = None + if not waiter.done(): + waiter.set_result(None) + + def feed_eof(self) -> None: + self._eof = True + self._release_waiter() + self._exception = None # Break cyclic references + + def feed_data(self, data: "WSMessage", size: "cython_int") -> None: + self._size += size + self._put_buffer((data, size)) + self._release_waiter() + if self._size > self._limit and not self._protocol._reading_paused: + self._protocol.pause_reading() + + async def read(self) -> WSMessage: + if not self._buffer and not self._eof: + assert not self._waiter + self._waiter = self._loop.create_future() + try: + await self._waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._waiter = None + raise + return self._read_from_buffer() + + def _read_from_buffer(self) -> WSMessage: + if self._buffer: + data, size = self._get_buffer() + self._size -= size + if self._size < self._limit and self._protocol._reading_paused: + self._protocol.resume_reading() + return data + if self._exception is not None: + raise self._exception + raise EofStream + + +class WebSocketReader: + def __init__( + self, queue: WebSocketDataQueue, max_msg_size: int, compress: bool = True + ) -> None: + self.queue = queue + self._max_msg_size = max_msg_size + + self._exc: Optional[Exception] = None + self._partial = bytearray() + self._state = READ_HEADER + + self._opcode: int = OP_CODE_NOT_SET + self._frame_fin = False + self._frame_opcode: int = OP_CODE_NOT_SET + self._payload_fragments: list[bytes] = [] + self._frame_payload_len = 0 + + self._tail: bytes = b"" + self._has_mask = False + self._frame_mask: Optional[bytes] = None + self._payload_bytes_to_read = 0 + self._payload_len_flag = 0 + self._compressed: int = COMPRESSED_NOT_SET + self._decompressobj: Optional[ZLibDecompressor] = None + self._compress = compress + + def feed_eof(self) -> None: + self.queue.feed_eof() + + # data can be bytearray on Windows because proactor event loop uses bytearray + # and asyncio types this to Union[bytes, bytearray, memoryview] so we need + # coerce data to bytes if it is not + def feed_data( + self, data: Union[bytes, bytearray, memoryview] + ) -> Tuple[bool, bytes]: + if type(data) is not bytes: + data = bytes(data) + + if self._exc is not None: + return True, data + + try: + self._feed_data(data) + except Exception as exc: + self._exc = exc + set_exception(self.queue, exc) + return EMPTY_FRAME_ERROR + + return EMPTY_FRAME + + def _handle_frame( + self, + fin: bool, + opcode: Union[int, cython_int], # Union intended: Cython pxd uses C int + payload: Union[bytes, bytearray], + compressed: Union[int, cython_int], # Union intended: Cython pxd uses C int + ) -> None: + msg: WSMessage + if opcode in {OP_CODE_TEXT, OP_CODE_BINARY, OP_CODE_CONTINUATION}: + # load text/binary + if not fin: + # got partial frame payload + if opcode != OP_CODE_CONTINUATION: + self._opcode = opcode + self._partial += payload + if self._max_msg_size and len(self._partial) >= self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Message size {len(self._partial)} " + f"exceeds limit {self._max_msg_size}", + ) + return + + has_partial = bool(self._partial) + if opcode == OP_CODE_CONTINUATION: + if self._opcode == OP_CODE_NOT_SET: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Continuation frame for non started message", + ) + opcode = self._opcode + self._opcode = OP_CODE_NOT_SET + # previous frame was non finished + # we should get continuation opcode + elif has_partial: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "The opcode in non-fin frame is expected " + f"to be zero, got {opcode!r}", + ) + + assembled_payload: Union[bytes, bytearray] + if has_partial: + assembled_payload = self._partial + payload + self._partial.clear() + else: + assembled_payload = payload + + if self._max_msg_size and len(assembled_payload) >= self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Message size {len(assembled_payload)} " + f"exceeds limit {self._max_msg_size}", + ) + + # Decompress process must to be done after all packets + # received. + if compressed: + if not self._decompressobj: + self._decompressobj = ZLibDecompressor(suppress_deflate_header=True) + # XXX: It's possible that the zlib backend (isal is known to + # do this, maybe others too?) will return max_length bytes, + # but internally buffer more data such that the payload is + # >max_length, so we return one extra byte and if we're able + # to do that, then the message is too big. + payload_merged = self._decompressobj.decompress_sync( + assembled_payload + WS_DEFLATE_TRAILING, + ( + self._max_msg_size + 1 + if self._max_msg_size + else self._max_msg_size + ), + ) + if self._max_msg_size and len(payload_merged) > self._max_msg_size: + raise WebSocketError( + WSCloseCode.MESSAGE_TOO_BIG, + f"Decompressed message exceeds size limit {self._max_msg_size}", + ) + elif type(assembled_payload) is bytes: + payload_merged = assembled_payload + else: + payload_merged = bytes(assembled_payload) + + if opcode == OP_CODE_TEXT: + try: + text = payload_merged.decode("utf-8") + except UnicodeDecodeError as exc: + raise WebSocketError( + WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" + ) from exc + + # XXX: The Text and Binary messages here can be a performance + # bottleneck, so we use tuple.__new__ to improve performance. + # This is not type safe, but many tests should fail in + # test_client_ws_functional.py if this is wrong. + self.queue.feed_data( + TUPLE_NEW(WSMessage, (WS_MSG_TYPE_TEXT, text, "")), + len(payload_merged), + ) + else: + self.queue.feed_data( + TUPLE_NEW(WSMessage, (WS_MSG_TYPE_BINARY, payload_merged, "")), + len(payload_merged), + ) + elif opcode == OP_CODE_CLOSE: + if len(payload) >= 2: + close_code = UNPACK_CLOSE_CODE(payload[:2])[0] + if close_code < 3000 and close_code not in ALLOWED_CLOSE_CODES: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Invalid close code: {close_code}", + ) + try: + close_message = payload[2:].decode("utf-8") + except UnicodeDecodeError as exc: + raise WebSocketError( + WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message" + ) from exc + msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, close_code, close_message)) + elif payload: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + f"Invalid close frame: {fin} {opcode} {payload!r}", + ) + else: + msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, 0, "")) + + self.queue.feed_data(msg, 0) + elif opcode == OP_CODE_PING: + msg = TUPLE_NEW(WSMessage, (WSMsgType.PING, payload, "")) + self.queue.feed_data(msg, len(payload)) + elif opcode == OP_CODE_PONG: + msg = TUPLE_NEW(WSMessage, (WSMsgType.PONG, payload, "")) + self.queue.feed_data(msg, len(payload)) + else: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, f"Unexpected opcode={opcode!r}" + ) + + def _feed_data(self, data: bytes) -> None: + """Return the next frame from the socket.""" + if self._tail: + data, self._tail = self._tail + data, b"" + + start_pos: int = 0 + data_len = len(data) + data_cstr = data + + while True: + # read header + if self._state == READ_HEADER: + if data_len - start_pos < 2: + break + first_byte = data_cstr[start_pos] + second_byte = data_cstr[start_pos + 1] + start_pos += 2 + + fin = (first_byte >> 7) & 1 + rsv1 = (first_byte >> 6) & 1 + rsv2 = (first_byte >> 5) & 1 + rsv3 = (first_byte >> 4) & 1 + opcode = first_byte & 0xF + + # frame-fin = %x0 ; more frames of this message follow + # / %x1 ; final frame of this message + # frame-rsv1 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # frame-rsv2 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # frame-rsv3 = %x0 ; + # 1 bit, MUST be 0 unless negotiated otherwise + # + # Remove rsv1 from this test for deflate development + if rsv2 or rsv3 or (rsv1 and not self._compress): + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received frame with non-zero reserved bits", + ) + + if opcode > 0x7 and fin == 0: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received fragmented control frame", + ) + + has_mask = (second_byte >> 7) & 1 + length = second_byte & 0x7F + + # Control frames MUST have a payload + # length of 125 bytes or less + if opcode > 0x7 and length > 125: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Control frame payload cannot be larger than 125 bytes", + ) + + # Set compress status if last package is FIN + # OR set compress status if this is first fragment + # Raise error if not first fragment with rsv1 = 0x1 + if self._frame_fin or self._compressed == COMPRESSED_NOT_SET: + self._compressed = COMPRESSED_TRUE if rsv1 else COMPRESSED_FALSE + elif rsv1: + raise WebSocketError( + WSCloseCode.PROTOCOL_ERROR, + "Received frame with non-zero reserved bits", + ) + + self._frame_fin = bool(fin) + self._frame_opcode = opcode + self._has_mask = bool(has_mask) + self._payload_len_flag = length + self._state = READ_PAYLOAD_LENGTH + + # read payload length + if self._state == READ_PAYLOAD_LENGTH: + len_flag = self._payload_len_flag + if len_flag == 126: + if data_len - start_pos < 2: + break + first_byte = data_cstr[start_pos] + second_byte = data_cstr[start_pos + 1] + start_pos += 2 + self._payload_bytes_to_read = first_byte << 8 | second_byte + elif len_flag > 126: + if data_len - start_pos < 8: + break + self._payload_bytes_to_read = UNPACK_LEN3(data, start_pos)[0] + start_pos += 8 + else: + self._payload_bytes_to_read = len_flag + + self._state = READ_PAYLOAD_MASK if self._has_mask else READ_PAYLOAD + + # read payload mask + if self._state == READ_PAYLOAD_MASK: + if data_len - start_pos < 4: + break + self._frame_mask = data_cstr[start_pos : start_pos + 4] + start_pos += 4 + self._state = READ_PAYLOAD + + if self._state == READ_PAYLOAD: + chunk_len = data_len - start_pos + if self._payload_bytes_to_read >= chunk_len: + f_end_pos = data_len + self._payload_bytes_to_read -= chunk_len + else: + f_end_pos = start_pos + self._payload_bytes_to_read + self._payload_bytes_to_read = 0 + + had_fragments = self._frame_payload_len + self._frame_payload_len += f_end_pos - start_pos + f_start_pos = start_pos + start_pos = f_end_pos + + if self._payload_bytes_to_read != 0: + # If we don't have a complete frame, we need to save the + # data for the next call to feed_data. + self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) + break + + payload: Union[bytes, bytearray] + if had_fragments: + # We have to join the payload fragments get the payload + self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos]) + if self._has_mask: + assert self._frame_mask is not None + payload_bytearray = bytearray(b"".join(self._payload_fragments)) + websocket_mask(self._frame_mask, payload_bytearray) + payload = payload_bytearray + else: + payload = b"".join(self._payload_fragments) + self._payload_fragments.clear() + elif self._has_mask: + assert self._frame_mask is not None + payload_bytearray = data_cstr[f_start_pos:f_end_pos] # type: ignore[assignment] + if type(payload_bytearray) is not bytearray: # pragma: no branch + # Cython will do the conversion for us + # but we need to do it for Python and we + # will always get here in Python + payload_bytearray = bytearray(payload_bytearray) + websocket_mask(self._frame_mask, payload_bytearray) + payload = payload_bytearray + else: + payload = data_cstr[f_start_pos:f_end_pos] + + self._handle_frame( + self._frame_fin, self._frame_opcode, payload, self._compressed + ) + self._frame_payload_len = 0 + self._state = READ_HEADER + + # XXX: Cython needs slices to be bounded, so we can't omit the slice end here. + self._tail = data_cstr[start_pos:data_len] if start_pos < data_len else b"" diff --git a/lib/python3.1/site-packages/aiohttp/_websocket/writer.py b/lib/python3.1/site-packages/aiohttp/_websocket/writer.py new file mode 100644 index 0000000000000000000000000000000000000000..19163f9afdf7b689e9e3f7959c0fc1daee096a4c --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/_websocket/writer.py @@ -0,0 +1,178 @@ +"""WebSocket protocol versions 13 and 8.""" + +import asyncio +import random +from functools import partial +from typing import Any, Final, Optional, Union + +from ..base_protocol import BaseProtocol +from ..client_exceptions import ClientConnectionResetError +from ..compression_utils import ZLibBackend, ZLibCompressor +from .helpers import ( + MASK_LEN, + MSG_SIZE, + PACK_CLOSE_CODE, + PACK_LEN1, + PACK_LEN2, + PACK_LEN3, + PACK_RANDBITS, + websocket_mask, +) +from .models import WS_DEFLATE_TRAILING, WSMsgType + +DEFAULT_LIMIT: Final[int] = 2**16 + +# For websockets, keeping latency low is extremely important as implementations +# generally expect to be able to send and receive messages quickly. We use a +# larger chunk size than the default to reduce the number of executor calls +# since the executor is a significant source of latency and overhead when +# the chunks are small. A size of 5KiB was chosen because it is also the +# same value python-zlib-ng choose to use as the threshold to release the GIL. + +WEBSOCKET_MAX_SYNC_CHUNK_SIZE = 5 * 1024 + + +class WebSocketWriter: + """WebSocket writer. + + The writer is responsible for sending messages to the client. It is + created by the protocol when a connection is established. The writer + should avoid implementing any application logic and should only be + concerned with the low-level details of the WebSocket protocol. + """ + + def __init__( + self, + protocol: BaseProtocol, + transport: asyncio.Transport, + *, + use_mask: bool = False, + limit: int = DEFAULT_LIMIT, + random: random.Random = random.Random(), + compress: int = 0, + notakeover: bool = False, + ) -> None: + """Initialize a WebSocket writer.""" + self.protocol = protocol + self.transport = transport + self.use_mask = use_mask + self.get_random_bits = partial(random.getrandbits, 32) + self.compress = compress + self.notakeover = notakeover + self._closing = False + self._limit = limit + self._output_size = 0 + self._compressobj: Any = None # actually compressobj + + async def send_frame( + self, message: bytes, opcode: int, compress: Optional[int] = None + ) -> None: + """Send a frame over the websocket with message as its payload.""" + if self._closing and not (opcode & WSMsgType.CLOSE): + raise ClientConnectionResetError("Cannot write to closing transport") + + # RSV are the reserved bits in the frame header. They are used to + # indicate that the frame is using an extension. + # https://datatracker.ietf.org/doc/html/rfc6455#section-5.2 + rsv = 0 + # Only compress larger packets (disabled) + # Does small packet needs to be compressed? + # if self.compress and opcode < 8 and len(message) > 124: + if (compress or self.compress) and opcode < 8: + # RSV1 (rsv = 0x40) is set for compressed frames + # https://datatracker.ietf.org/doc/html/rfc7692#section-7.2.3.1 + rsv = 0x40 + + if compress: + # Do not set self._compress if compressing is for this frame + compressobj = self._make_compress_obj(compress) + else: # self.compress + if not self._compressobj: + self._compressobj = self._make_compress_obj(self.compress) + compressobj = self._compressobj + + message = ( + await compressobj.compress(message) + + compressobj.flush( + ZLibBackend.Z_FULL_FLUSH + if self.notakeover + else ZLibBackend.Z_SYNC_FLUSH + ) + ).removesuffix(WS_DEFLATE_TRAILING) + # Its critical that we do not return control to the event + # loop until we have finished sending all the compressed + # data. Otherwise we could end up mixing compressed frames + # if there are multiple coroutines compressing data. + + msg_length = len(message) + + use_mask = self.use_mask + mask_bit = 0x80 if use_mask else 0 + + # Depending on the message length, the header is assembled differently. + # The first byte is reserved for the opcode and the RSV bits. + first_byte = 0x80 | rsv | opcode + if msg_length < 126: + header = PACK_LEN1(first_byte, msg_length | mask_bit) + header_len = 2 + elif msg_length < 65536: + header = PACK_LEN2(first_byte, 126 | mask_bit, msg_length) + header_len = 4 + else: + header = PACK_LEN3(first_byte, 127 | mask_bit, msg_length) + header_len = 10 + + if self.transport.is_closing(): + raise ClientConnectionResetError("Cannot write to closing transport") + + # https://datatracker.ietf.org/doc/html/rfc6455#section-5.3 + # If we are using a mask, we need to generate it randomly + # and apply it to the message before sending it. A mask is + # a 32-bit value that is applied to the message using a + # bitwise XOR operation. It is used to prevent certain types + # of attacks on the websocket protocol. The mask is only used + # when aiohttp is acting as a client. Servers do not use a mask. + if use_mask: + mask = PACK_RANDBITS(self.get_random_bits()) + message = bytearray(message) + websocket_mask(mask, message) + self.transport.write(header + mask + message) + self._output_size += MASK_LEN + elif msg_length > MSG_SIZE: + self.transport.write(header) + self.transport.write(message) + else: + self.transport.write(header + message) + + self._output_size += header_len + msg_length + + # It is safe to return control to the event loop when using compression + # after this point as we have already sent or buffered all the data. + + # Once we have written output_size up to the limit, we call the + # drain helper which waits for the transport to be ready to accept + # more data. This is a flow control mechanism to prevent the buffer + # from growing too large. The drain helper will return right away + # if the writer is not paused. + if self._output_size > self._limit: + self._output_size = 0 + if self.protocol._paused: + await self.protocol._drain_helper() + + def _make_compress_obj(self, compress: int) -> ZLibCompressor: + return ZLibCompressor( + level=ZLibBackend.Z_BEST_SPEED, + wbits=-compress, + max_sync_chunk_size=WEBSOCKET_MAX_SYNC_CHUNK_SIZE, + ) + + async def close(self, code: int = 1000, message: Union[bytes, str] = b"") -> None: + """Close the websocket, sending the specified code and message.""" + if isinstance(message, str): + message = message.encode("utf-8") + try: + await self.send_frame( + PACK_CLOSE_CODE(code) + message, opcode=WSMsgType.CLOSE + ) + finally: + self._closing = True diff --git a/lib/python3.1/site-packages/aiohttp/abc.py b/lib/python3.1/site-packages/aiohttp/abc.py new file mode 100644 index 0000000000000000000000000000000000000000..2574ff936219593a5ea063302e23cca2c4b0a02c --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/abc.py @@ -0,0 +1,268 @@ +import asyncio +import logging +import socket +from abc import ABC, abstractmethod +from collections.abc import Sized +from http.cookies import BaseCookie, Morsel +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Dict, + Generator, + Iterable, + List, + Optional, + Sequence, + Tuple, + TypedDict, + Union, +) + +from multidict import CIMultiDict +from yarl import URL + +from ._cookie_helpers import parse_set_cookie_headers +from .typedefs import LooseCookies + +if TYPE_CHECKING: + from .web_app import Application + from .web_exceptions import HTTPException + from .web_request import BaseRequest, Request + from .web_response import StreamResponse +else: + BaseRequest = Request = Application = StreamResponse = None + HTTPException = None + + +class AbstractRouter(ABC): + def __init__(self) -> None: + self._frozen = False + + def post_init(self, app: Application) -> None: + """Post init stage. + + Not an abstract method for sake of backward compatibility, + but if the router wants to be aware of the application + it can override this. + """ + + @property + def frozen(self) -> bool: + return self._frozen + + def freeze(self) -> None: + """Freeze router.""" + self._frozen = True + + @abstractmethod + async def resolve(self, request: Request) -> "AbstractMatchInfo": + """Return MATCH_INFO for given request""" + + +class AbstractMatchInfo(ABC): + + __slots__ = () + + @property # pragma: no branch + @abstractmethod + def handler(self) -> Callable[[Request], Awaitable[StreamResponse]]: + """Execute matched request handler""" + + @property + @abstractmethod + def expect_handler( + self, + ) -> Callable[[Request], Awaitable[Optional[StreamResponse]]]: + """Expect handler for 100-continue processing""" + + @property # pragma: no branch + @abstractmethod + def http_exception(self) -> Optional[HTTPException]: + """HTTPException instance raised on router's resolving, or None""" + + @abstractmethod # pragma: no branch + def get_info(self) -> Dict[str, Any]: + """Return a dict with additional info useful for introspection""" + + @property # pragma: no branch + @abstractmethod + def apps(self) -> Tuple[Application, ...]: + """Stack of nested applications. + + Top level application is left-most element. + + """ + + @abstractmethod + def add_app(self, app: Application) -> None: + """Add application to the nested apps stack.""" + + @abstractmethod + def freeze(self) -> None: + """Freeze the match info. + + The method is called after route resolution. + + After the call .add_app() is forbidden. + + """ + + +class AbstractView(ABC): + """Abstract class based view.""" + + def __init__(self, request: Request) -> None: + self._request = request + + @property + def request(self) -> Request: + """Request instance.""" + return self._request + + @abstractmethod + def __await__(self) -> Generator[Any, None, StreamResponse]: + """Execute the view handler.""" + + +class ResolveResult(TypedDict): + """Resolve result. + + This is the result returned from an AbstractResolver's + resolve method. + + :param hostname: The hostname that was provided. + :param host: The IP address that was resolved. + :param port: The port that was resolved. + :param family: The address family that was resolved. + :param proto: The protocol that was resolved. + :param flags: The flags that were resolved. + """ + + hostname: str + host: str + port: int + family: int + proto: int + flags: int + + +class AbstractResolver(ABC): + """Abstract DNS resolver.""" + + @abstractmethod + async def resolve( + self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET + ) -> List[ResolveResult]: + """Return IP address for given hostname""" + + @abstractmethod + async def close(self) -> None: + """Release resolver""" + + +if TYPE_CHECKING: + IterableBase = Iterable[Morsel[str]] +else: + IterableBase = Iterable + + +ClearCookiePredicate = Callable[["Morsel[str]"], bool] + + +class AbstractCookieJar(Sized, IterableBase): + """Abstract Cookie Jar.""" + + def __init__(self, *, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: + self._loop = loop or asyncio.get_running_loop() + + @property + @abstractmethod + def quote_cookie(self) -> bool: + """Return True if cookies should be quoted.""" + + @abstractmethod + def clear(self, predicate: Optional[ClearCookiePredicate] = None) -> None: + """Clear all cookies if no predicate is passed.""" + + @abstractmethod + def clear_domain(self, domain: str) -> None: + """Clear all cookies for domain and all subdomains.""" + + @abstractmethod + def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None: + """Update cookies.""" + + def update_cookies_from_headers( + self, headers: Sequence[str], response_url: URL + ) -> None: + """Update cookies from raw Set-Cookie headers.""" + if headers and (cookies_to_update := parse_set_cookie_headers(headers)): + self.update_cookies(cookies_to_update, response_url) + + @abstractmethod + def filter_cookies(self, request_url: URL) -> "BaseCookie[str]": + """Return the jar's cookies filtered by their attributes.""" + + +class AbstractStreamWriter(ABC): + """Abstract stream writer.""" + + buffer_size: int = 0 + output_size: int = 0 + length: Optional[int] = 0 + + @abstractmethod + async def write(self, chunk: Union[bytes, bytearray, memoryview]) -> None: + """Write chunk into stream.""" + + @abstractmethod + async def write_eof(self, chunk: bytes = b"") -> None: + """Write last chunk.""" + + @abstractmethod + async def drain(self) -> None: + """Flush the write buffer.""" + + @abstractmethod + def enable_compression( + self, encoding: str = "deflate", strategy: Optional[int] = None + ) -> None: + """Enable HTTP body compression""" + + @abstractmethod + def enable_chunking(self) -> None: + """Enable HTTP chunked mode""" + + @abstractmethod + async def write_headers( + self, status_line: str, headers: "CIMultiDict[str]" + ) -> None: + """Write HTTP headers""" + + def send_headers(self) -> None: + """Force sending buffered headers if not already sent. + + Required only if write_headers() buffers headers instead of sending immediately. + For backwards compatibility, this method does nothing by default. + """ + + +class AbstractAccessLogger(ABC): + """Abstract writer to access log.""" + + __slots__ = ("logger", "log_format") + + def __init__(self, logger: logging.Logger, log_format: str) -> None: + self.logger = logger + self.log_format = log_format + + @abstractmethod + def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None: + """Emit log to logger.""" + + @property + def enabled(self) -> bool: + """Check if logger is enabled.""" + return True diff --git a/lib/python3.1/site-packages/aiohttp/base_protocol.py b/lib/python3.1/site-packages/aiohttp/base_protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..b0a67ed6ff68ca5bc48be9ac472ee755369b2720 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/base_protocol.py @@ -0,0 +1,100 @@ +import asyncio +from typing import Optional, cast + +from .client_exceptions import ClientConnectionResetError +from .helpers import set_exception +from .tcp_helpers import tcp_nodelay + + +class BaseProtocol(asyncio.Protocol): + __slots__ = ( + "_loop", + "_paused", + "_drain_waiter", + "_connection_lost", + "_reading_paused", + "transport", + ) + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + self._loop: asyncio.AbstractEventLoop = loop + self._paused = False + self._drain_waiter: Optional[asyncio.Future[None]] = None + self._reading_paused = False + + self.transport: Optional[asyncio.Transport] = None + + @property + def connected(self) -> bool: + """Return True if the connection is open.""" + return self.transport is not None + + @property + def writing_paused(self) -> bool: + return self._paused + + def pause_writing(self) -> None: + assert not self._paused + self._paused = True + + def resume_writing(self) -> None: + assert self._paused + self._paused = False + + waiter = self._drain_waiter + if waiter is not None: + self._drain_waiter = None + if not waiter.done(): + waiter.set_result(None) + + def pause_reading(self) -> None: + if not self._reading_paused and self.transport is not None: + try: + self.transport.pause_reading() + except (AttributeError, NotImplementedError, RuntimeError): + pass + self._reading_paused = True + + def resume_reading(self) -> None: + if self._reading_paused and self.transport is not None: + try: + self.transport.resume_reading() + except (AttributeError, NotImplementedError, RuntimeError): + pass + self._reading_paused = False + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + tr = cast(asyncio.Transport, transport) + tcp_nodelay(tr, True) + self.transport = tr + + def connection_lost(self, exc: Optional[BaseException]) -> None: + # Wake up the writer if currently paused. + self.transport = None + if not self._paused: + return + waiter = self._drain_waiter + if waiter is None: + return + self._drain_waiter = None + if waiter.done(): + return + if exc is None: + waiter.set_result(None) + else: + set_exception( + waiter, + ConnectionError("Connection lost"), + exc, + ) + + async def _drain_helper(self) -> None: + if self.transport is None: + raise ClientConnectionResetError("Connection lost") + if not self._paused: + return + waiter = self._drain_waiter + if waiter is None: + waiter = self._loop.create_future() + self._drain_waiter = waiter + await asyncio.shield(waiter) diff --git a/lib/python3.1/site-packages/aiohttp/client.py b/lib/python3.1/site-packages/aiohttp/client.py new file mode 100644 index 0000000000000000000000000000000000000000..0c72d5948ce806d8e633f80da3d9a965aa86cf20 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/client.py @@ -0,0 +1,1613 @@ +"""HTTP Client for asyncio.""" + +import asyncio +import base64 +import hashlib +import json +import os +import sys +import traceback +import warnings +from contextlib import suppress +from types import TracebackType +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Coroutine, + Final, + FrozenSet, + Generator, + Generic, + Iterable, + List, + Mapping, + Optional, + Sequence, + Set, + Tuple, + Type, + TypedDict, + TypeVar, + Union, +) + +import attr +from multidict import CIMultiDict, MultiDict, MultiDictProxy, istr +from yarl import URL + +from . import hdrs, http, payload +from ._websocket.reader import WebSocketDataQueue +from .abc import AbstractCookieJar +from .client_exceptions import ( + ClientConnectionError, + ClientConnectionResetError, + ClientConnectorCertificateError, + ClientConnectorDNSError, + ClientConnectorError, + ClientConnectorSSLError, + ClientError, + ClientHttpProxyError, + ClientOSError, + ClientPayloadError, + ClientProxyConnectionError, + ClientResponseError, + ClientSSLError, + ConnectionTimeoutError, + ContentTypeError, + InvalidURL, + InvalidUrlClientError, + InvalidUrlRedirectClientError, + NonHttpUrlClientError, + NonHttpUrlRedirectClientError, + RedirectClientError, + ServerConnectionError, + ServerDisconnectedError, + ServerFingerprintMismatch, + ServerTimeoutError, + SocketTimeoutError, + TooManyRedirects, + WSMessageTypeError, + WSServerHandshakeError, +) +from .client_middlewares import ClientMiddlewareType, build_client_middlewares +from .client_reqrep import ( + ClientRequest as ClientRequest, + ClientResponse as ClientResponse, + Fingerprint as Fingerprint, + RequestInfo as RequestInfo, + _merge_ssl_params, +) +from .client_ws import ( + DEFAULT_WS_CLIENT_TIMEOUT, + ClientWebSocketResponse as ClientWebSocketResponse, + ClientWSTimeout as ClientWSTimeout, +) +from .connector import ( + HTTP_AND_EMPTY_SCHEMA_SET, + BaseConnector as BaseConnector, + NamedPipeConnector as NamedPipeConnector, + TCPConnector as TCPConnector, + UnixConnector as UnixConnector, +) +from .cookiejar import CookieJar +from .helpers import ( + _SENTINEL, + DEBUG, + EMPTY_BODY_METHODS, + BasicAuth, + TimeoutHandle, + get_env_proxy_for_url, + sentinel, + strip_auth_from_url, +) +from .http import WS_KEY, HttpVersion, WebSocketReader, WebSocketWriter +from .http_websocket import WSHandshakeError, ws_ext_gen, ws_ext_parse +from .tracing import Trace, TraceConfig +from .typedefs import JSONEncoder, LooseCookies, LooseHeaders, Query, StrOrURL + +__all__ = ( + # client_exceptions + "ClientConnectionError", + "ClientConnectionResetError", + "ClientConnectorCertificateError", + "ClientConnectorDNSError", + "ClientConnectorError", + "ClientConnectorSSLError", + "ClientError", + "ClientHttpProxyError", + "ClientOSError", + "ClientPayloadError", + "ClientProxyConnectionError", + "ClientResponseError", + "ClientSSLError", + "ConnectionTimeoutError", + "ContentTypeError", + "InvalidURL", + "InvalidUrlClientError", + "RedirectClientError", + "NonHttpUrlClientError", + "InvalidUrlRedirectClientError", + "NonHttpUrlRedirectClientError", + "ServerConnectionError", + "ServerDisconnectedError", + "ServerFingerprintMismatch", + "ServerTimeoutError", + "SocketTimeoutError", + "TooManyRedirects", + "WSServerHandshakeError", + # client_reqrep + "ClientRequest", + "ClientResponse", + "Fingerprint", + "RequestInfo", + # connector + "BaseConnector", + "TCPConnector", + "UnixConnector", + "NamedPipeConnector", + # client_ws + "ClientWebSocketResponse", + # client + "ClientSession", + "ClientTimeout", + "ClientWSTimeout", + "request", + "WSMessageTypeError", +) + + +if TYPE_CHECKING: + from ssl import SSLContext +else: + SSLContext = None + +if sys.version_info >= (3, 11) and TYPE_CHECKING: + from typing import Unpack + + +class _RequestOptions(TypedDict, total=False): + params: Query + data: Any + json: Any + cookies: Union[LooseCookies, None] + headers: Union[LooseHeaders, None] + skip_auto_headers: Union[Iterable[str], None] + auth: Union[BasicAuth, None] + allow_redirects: bool + max_redirects: int + compress: Union[str, bool, None] + chunked: Union[bool, None] + expect100: bool + raise_for_status: Union[None, bool, Callable[[ClientResponse], Awaitable[None]]] + read_until_eof: bool + proxy: Union[StrOrURL, None] + proxy_auth: Union[BasicAuth, None] + timeout: "Union[ClientTimeout, _SENTINEL, None]" + ssl: Union[SSLContext, bool, Fingerprint] + server_hostname: Union[str, None] + proxy_headers: Union[LooseHeaders, None] + trace_request_ctx: Union[Mapping[str, Any], None] + read_bufsize: Union[int, None] + auto_decompress: Union[bool, None] + max_line_size: Union[int, None] + max_field_size: Union[int, None] + middlewares: Optional[Sequence[ClientMiddlewareType]] + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ClientTimeout: + total: Optional[float] = None + connect: Optional[float] = None + sock_read: Optional[float] = None + sock_connect: Optional[float] = None + ceil_threshold: float = 5 + + # pool_queue_timeout: Optional[float] = None + # dns_resolution_timeout: Optional[float] = None + # socket_connect_timeout: Optional[float] = None + # connection_acquiring_timeout: Optional[float] = None + # new_connection_timeout: Optional[float] = None + # http_header_timeout: Optional[float] = None + # response_body_timeout: Optional[float] = None + + # to create a timeout specific for a single request, either + # - create a completely new one to overwrite the default + # - or use http://www.attrs.org/en/stable/api.html#attr.evolve + # to overwrite the defaults + + +# 5 Minute default read timeout +DEFAULT_TIMEOUT: Final[ClientTimeout] = ClientTimeout(total=5 * 60, sock_connect=30) + +# https://www.rfc-editor.org/rfc/rfc9110#section-9.2.2 +IDEMPOTENT_METHODS = frozenset({"GET", "HEAD", "OPTIONS", "TRACE", "PUT", "DELETE"}) + +_RetType = TypeVar("_RetType", ClientResponse, ClientWebSocketResponse) +_CharsetResolver = Callable[[ClientResponse, bytes], str] + + +class ClientSession: + """First-class interface for making HTTP requests.""" + + ATTRS = frozenset( + [ + "_base_url", + "_base_url_origin", + "_source_traceback", + "_connector", + "_loop", + "_cookie_jar", + "_connector_owner", + "_default_auth", + "_version", + "_json_serialize", + "_requote_redirect_url", + "_timeout", + "_raise_for_status", + "_auto_decompress", + "_trust_env", + "_default_headers", + "_skip_auto_headers", + "_request_class", + "_response_class", + "_ws_response_class", + "_trace_configs", + "_read_bufsize", + "_max_line_size", + "_max_field_size", + "_resolve_charset", + "_default_proxy", + "_default_proxy_auth", + "_retry_connection", + "_middlewares", + "requote_redirect_url", + ] + ) + + _source_traceback: Optional[traceback.StackSummary] = None + _connector: Optional[BaseConnector] = None + + def __init__( + self, + base_url: Optional[StrOrURL] = None, + *, + connector: Optional[BaseConnector] = None, + loop: Optional[asyncio.AbstractEventLoop] = None, + cookies: Optional[LooseCookies] = None, + headers: Optional[LooseHeaders] = None, + proxy: Optional[StrOrURL] = None, + proxy_auth: Optional[BasicAuth] = None, + skip_auto_headers: Optional[Iterable[str]] = None, + auth: Optional[BasicAuth] = None, + json_serialize: JSONEncoder = json.dumps, + request_class: Type[ClientRequest] = ClientRequest, + response_class: Type[ClientResponse] = ClientResponse, + ws_response_class: Type[ClientWebSocketResponse] = ClientWebSocketResponse, + version: HttpVersion = http.HttpVersion11, + cookie_jar: Optional[AbstractCookieJar] = None, + connector_owner: bool = True, + raise_for_status: Union[ + bool, Callable[[ClientResponse], Awaitable[None]] + ] = False, + read_timeout: Union[float, _SENTINEL] = sentinel, + conn_timeout: Optional[float] = None, + timeout: Union[object, ClientTimeout] = sentinel, + auto_decompress: bool = True, + trust_env: bool = False, + requote_redirect_url: bool = True, + trace_configs: Optional[List[TraceConfig]] = None, + read_bufsize: int = 2**16, + max_line_size: int = 8190, + max_field_size: int = 8190, + fallback_charset_resolver: _CharsetResolver = lambda r, b: "utf-8", + middlewares: Sequence[ClientMiddlewareType] = (), + ssl_shutdown_timeout: Union[_SENTINEL, None, float] = sentinel, + ) -> None: + # We initialise _connector to None immediately, as it's referenced in __del__() + # and could cause issues if an exception occurs during initialisation. + self._connector: Optional[BaseConnector] = None + + if loop is None: + if connector is not None: + loop = connector._loop + + loop = loop or asyncio.get_running_loop() + + if base_url is None or isinstance(base_url, URL): + self._base_url: Optional[URL] = base_url + self._base_url_origin = None if base_url is None else base_url.origin() + else: + self._base_url = URL(base_url) + self._base_url_origin = self._base_url.origin() + assert self._base_url.absolute, "Only absolute URLs are supported" + if self._base_url is not None and not self._base_url.path.endswith("/"): + raise ValueError("base_url must have a trailing '/'") + + if timeout is sentinel or timeout is None: + self._timeout = DEFAULT_TIMEOUT + if read_timeout is not sentinel: + warnings.warn( + "read_timeout is deprecated, use timeout argument instead", + DeprecationWarning, + stacklevel=2, + ) + self._timeout = attr.evolve(self._timeout, total=read_timeout) + if conn_timeout is not None: + self._timeout = attr.evolve(self._timeout, connect=conn_timeout) + warnings.warn( + "conn_timeout is deprecated, use timeout argument instead", + DeprecationWarning, + stacklevel=2, + ) + else: + if not isinstance(timeout, ClientTimeout): + raise ValueError( + f"timeout parameter cannot be of {type(timeout)} type, " + "please use 'timeout=ClientTimeout(...)'", + ) + self._timeout = timeout + if read_timeout is not sentinel: + raise ValueError( + "read_timeout and timeout parameters " + "conflict, please setup " + "timeout.read" + ) + if conn_timeout is not None: + raise ValueError( + "conn_timeout and timeout parameters " + "conflict, please setup " + "timeout.connect" + ) + + if ssl_shutdown_timeout is not sentinel: + warnings.warn( + "The ssl_shutdown_timeout parameter is deprecated and will be removed in aiohttp 4.0", + DeprecationWarning, + stacklevel=2, + ) + + if connector is None: + connector = TCPConnector( + loop=loop, ssl_shutdown_timeout=ssl_shutdown_timeout + ) + + if connector._loop is not loop: + raise RuntimeError("Session and connector has to use same event loop") + + self._loop = loop + + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + if cookie_jar is None: + cookie_jar = CookieJar(loop=loop) + self._cookie_jar = cookie_jar + + if cookies: + self._cookie_jar.update_cookies(cookies) + + self._connector = connector + self._connector_owner = connector_owner + self._default_auth = auth + self._version = version + self._json_serialize = json_serialize + self._raise_for_status = raise_for_status + self._auto_decompress = auto_decompress + self._trust_env = trust_env + self._requote_redirect_url = requote_redirect_url + self._read_bufsize = read_bufsize + self._max_line_size = max_line_size + self._max_field_size = max_field_size + + # Convert to list of tuples + if headers: + real_headers: CIMultiDict[str] = CIMultiDict(headers) + else: + real_headers = CIMultiDict() + self._default_headers: CIMultiDict[str] = real_headers + if skip_auto_headers is not None: + self._skip_auto_headers = frozenset(istr(i) for i in skip_auto_headers) + else: + self._skip_auto_headers = frozenset() + + self._request_class = request_class + self._response_class = response_class + self._ws_response_class = ws_response_class + + self._trace_configs = trace_configs or [] + for trace_config in self._trace_configs: + trace_config.freeze() + + self._resolve_charset = fallback_charset_resolver + + self._default_proxy = proxy + self._default_proxy_auth = proxy_auth + self._retry_connection: bool = True + self._middlewares = middlewares + + def __init_subclass__(cls: Type["ClientSession"]) -> None: + warnings.warn( + "Inheritance class {} from ClientSession " + "is discouraged".format(cls.__name__), + DeprecationWarning, + stacklevel=2, + ) + + if DEBUG: + + def __setattr__(self, name: str, val: Any) -> None: + if name not in self.ATTRS: + warnings.warn( + "Setting custom ClientSession.{} attribute " + "is discouraged".format(name), + DeprecationWarning, + stacklevel=2, + ) + super().__setattr__(name, val) + + def __del__(self, _warnings: Any = warnings) -> None: + if not self.closed: + kwargs = {"source": self} + _warnings.warn( + f"Unclosed client session {self!r}", ResourceWarning, **kwargs + ) + context = {"client_session": self, "message": "Unclosed client session"} + if self._source_traceback is not None: + context["source_traceback"] = self._source_traceback + self._loop.call_exception_handler(context) + + if sys.version_info >= (3, 11) and TYPE_CHECKING: + + def request( + self, + method: str, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + else: + + def request( + self, method: str, url: StrOrURL, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP request.""" + return _RequestContextManager(self._request(method, url, **kwargs)) + + def _build_url(self, str_or_url: StrOrURL) -> URL: + url = URL(str_or_url) + if self._base_url and not url.absolute: + return self._base_url.join(url) + return url + + async def _request( + self, + method: str, + str_or_url: StrOrURL, + *, + params: Query = None, + data: Any = None, + json: Any = None, + cookies: Optional[LooseCookies] = None, + headers: Optional[LooseHeaders] = None, + skip_auto_headers: Optional[Iterable[str]] = None, + auth: Optional[BasicAuth] = None, + allow_redirects: bool = True, + max_redirects: int = 10, + compress: Union[str, bool, None] = None, + chunked: Optional[bool] = None, + expect100: bool = False, + raise_for_status: Union[ + None, bool, Callable[[ClientResponse], Awaitable[None]] + ] = None, + read_until_eof: bool = True, + proxy: Optional[StrOrURL] = None, + proxy_auth: Optional[BasicAuth] = None, + timeout: Union[ClientTimeout, _SENTINEL] = sentinel, + verify_ssl: Optional[bool] = None, + fingerprint: Optional[bytes] = None, + ssl_context: Optional[SSLContext] = None, + ssl: Union[SSLContext, bool, Fingerprint] = True, + server_hostname: Optional[str] = None, + proxy_headers: Optional[LooseHeaders] = None, + trace_request_ctx: Optional[Mapping[str, Any]] = None, + read_bufsize: Optional[int] = None, + auto_decompress: Optional[bool] = None, + max_line_size: Optional[int] = None, + max_field_size: Optional[int] = None, + middlewares: Optional[Sequence[ClientMiddlewareType]] = None, + ) -> ClientResponse: + + # NOTE: timeout clamps existing connect and read timeouts. We cannot + # set the default to None because we need to detect if the user wants + # to use the existing timeouts by setting timeout to None. + + if self.closed: + raise RuntimeError("Session is closed") + + ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint) + + if data is not None and json is not None: + raise ValueError( + "data and json parameters can not be used at the same time" + ) + elif json is not None: + data = payload.JsonPayload(json, dumps=self._json_serialize) + + if not isinstance(chunked, bool) and chunked is not None: + warnings.warn("Chunk size is deprecated #1615", DeprecationWarning) + + redirects = 0 + history: List[ClientResponse] = [] + version = self._version + params = params or {} + + # Merge with default headers and transform to CIMultiDict + headers = self._prepare_headers(headers) + + try: + url = self._build_url(str_or_url) + except ValueError as e: + raise InvalidUrlClientError(str_or_url) from e + + assert self._connector is not None + if url.scheme not in self._connector.allowed_protocol_schema_set: + raise NonHttpUrlClientError(url) + + skip_headers: Optional[Iterable[istr]] + if skip_auto_headers is not None: + skip_headers = { + istr(i) for i in skip_auto_headers + } | self._skip_auto_headers + elif self._skip_auto_headers: + skip_headers = self._skip_auto_headers + else: + skip_headers = None + + if proxy is None: + proxy = self._default_proxy + if proxy_auth is None: + proxy_auth = self._default_proxy_auth + + if proxy is None: + proxy_headers = None + else: + proxy_headers = self._prepare_headers(proxy_headers) + try: + proxy = URL(proxy) + except ValueError as e: + raise InvalidURL(proxy) from e + + if timeout is sentinel: + real_timeout: ClientTimeout = self._timeout + else: + if not isinstance(timeout, ClientTimeout): + real_timeout = ClientTimeout(total=timeout) + else: + real_timeout = timeout + # timeout is cumulative for all request operations + # (request, redirects, responses, data consuming) + tm = TimeoutHandle( + self._loop, real_timeout.total, ceil_threshold=real_timeout.ceil_threshold + ) + handle = tm.start() + + if read_bufsize is None: + read_bufsize = self._read_bufsize + + if auto_decompress is None: + auto_decompress = self._auto_decompress + + if max_line_size is None: + max_line_size = self._max_line_size + + if max_field_size is None: + max_field_size = self._max_field_size + + traces = [ + Trace( + self, + trace_config, + trace_config.trace_config_ctx(trace_request_ctx=trace_request_ctx), + ) + for trace_config in self._trace_configs + ] + + for trace in traces: + await trace.send_request_start(method, url.update_query(params), headers) + + timer = tm.timer() + try: + with timer: + # https://www.rfc-editor.org/rfc/rfc9112.html#name-retrying-requests + retry_persistent_connection = ( + self._retry_connection and method in IDEMPOTENT_METHODS + ) + while True: + url, auth_from_url = strip_auth_from_url(url) + if not url.raw_host: + # NOTE: Bail early, otherwise, causes `InvalidURL` through + # NOTE: `self._request_class()` below. + err_exc_cls = ( + InvalidUrlRedirectClientError + if redirects + else InvalidUrlClientError + ) + raise err_exc_cls(url) + # If `auth` was passed for an already authenticated URL, + # disallow only if this is the initial URL; this is to avoid issues + # with sketchy redirects that are not the caller's responsibility + if not history and (auth and auth_from_url): + raise ValueError( + "Cannot combine AUTH argument with " + "credentials encoded in URL" + ) + + # Override the auth with the one from the URL only if we + # have no auth, or if we got an auth from a redirect URL + if auth is None or (history and auth_from_url is not None): + auth = auth_from_url + + if ( + auth is None + and self._default_auth + and ( + not self._base_url or self._base_url_origin == url.origin() + ) + ): + auth = self._default_auth + # It would be confusing if we support explicit + # Authorization header with auth argument + if ( + headers is not None + and auth is not None + and hdrs.AUTHORIZATION in headers + ): + raise ValueError( + "Cannot combine AUTHORIZATION header " + "with AUTH argument or credentials " + "encoded in URL" + ) + + all_cookies = self._cookie_jar.filter_cookies(url) + + if cookies is not None: + tmp_cookie_jar = CookieJar( + quote_cookie=self._cookie_jar.quote_cookie + ) + tmp_cookie_jar.update_cookies(cookies) + req_cookies = tmp_cookie_jar.filter_cookies(url) + if req_cookies: + all_cookies.load(req_cookies) + + proxy_: Optional[URL] = None + if proxy is not None: + proxy_ = URL(proxy) + elif self._trust_env: + with suppress(LookupError): + proxy_, proxy_auth = await asyncio.to_thread( + get_env_proxy_for_url, url + ) + + req = self._request_class( + method, + url, + params=params, + headers=headers, + skip_auto_headers=skip_headers, + data=data, + cookies=all_cookies, + auth=auth, + version=version, + compress=compress, + chunked=chunked, + expect100=expect100, + loop=self._loop, + response_class=self._response_class, + proxy=proxy_, + proxy_auth=proxy_auth, + timer=timer, + session=self, + ssl=ssl if ssl is not None else True, + server_hostname=server_hostname, + proxy_headers=proxy_headers, + traces=traces, + trust_env=self.trust_env, + ) + + async def _connect_and_send_request( + req: ClientRequest, + ) -> ClientResponse: + # connection timeout + assert self._connector is not None + try: + conn = await self._connector.connect( + req, traces=traces, timeout=real_timeout + ) + except asyncio.TimeoutError as exc: + raise ConnectionTimeoutError( + f"Connection timeout to host {req.url}" + ) from exc + + assert conn.protocol is not None + conn.protocol.set_response_params( + timer=timer, + skip_payload=req.method in EMPTY_BODY_METHODS, + read_until_eof=read_until_eof, + auto_decompress=auto_decompress, + read_timeout=real_timeout.sock_read, + read_bufsize=read_bufsize, + timeout_ceil_threshold=self._connector._timeout_ceil_threshold, + max_line_size=max_line_size, + max_field_size=max_field_size, + ) + try: + resp = await req.send(conn) + try: + await resp.start(conn) + except BaseException: + resp.close() + raise + except BaseException: + conn.close() + raise + return resp + + # Apply middleware (if any) - per-request middleware overrides session middleware + effective_middlewares = ( + self._middlewares if middlewares is None else middlewares + ) + + if effective_middlewares: + handler = build_client_middlewares( + _connect_and_send_request, effective_middlewares + ) + else: + handler = _connect_and_send_request + + try: + resp = await handler(req) + # Client connector errors should not be retried + except ( + ConnectionTimeoutError, + ClientConnectorError, + ClientConnectorCertificateError, + ClientConnectorSSLError, + ): + raise + except (ClientOSError, ServerDisconnectedError): + if retry_persistent_connection: + retry_persistent_connection = False + continue + raise + except ClientError: + raise + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise ClientOSError(*exc.args) from exc + + # Update cookies from raw headers to preserve duplicates + if resp._raw_cookie_headers: + self._cookie_jar.update_cookies_from_headers( + resp._raw_cookie_headers, resp.url + ) + + # redirects + if resp.status in (301, 302, 303, 307, 308) and allow_redirects: + + for trace in traces: + await trace.send_request_redirect( + method, url.update_query(params), headers, resp + ) + + redirects += 1 + history.append(resp) + if max_redirects and redirects >= max_redirects: + if req._body is not None: + await req._body.close() + resp.close() + raise TooManyRedirects( + history[0].request_info, tuple(history) + ) + + # For 301 and 302, mimic IE, now changed in RFC + # https://github.com/kennethreitz/requests/pull/269 + if (resp.status == 303 and resp.method != hdrs.METH_HEAD) or ( + resp.status in (301, 302) and resp.method == hdrs.METH_POST + ): + method = hdrs.METH_GET + data = None + if headers.get(hdrs.CONTENT_LENGTH): + headers.pop(hdrs.CONTENT_LENGTH) + else: + # For 307/308, always preserve the request body + # For 301/302 with non-POST methods, preserve the request body + # https://www.rfc-editor.org/rfc/rfc9110#section-15.4.3-3.1 + # Use the existing payload to avoid recreating it from a potentially consumed file + data = req._body + + r_url = resp.headers.get(hdrs.LOCATION) or resp.headers.get( + hdrs.URI + ) + if r_url is None: + # see github.com/aio-libs/aiohttp/issues/2022 + break + else: + # reading from correct redirection + # response is forbidden + resp.release() + + try: + parsed_redirect_url = URL( + r_url, encoded=not self._requote_redirect_url + ) + except ValueError as e: + if req._body is not None: + await req._body.close() + resp.close() + raise InvalidUrlRedirectClientError( + r_url, + "Server attempted redirecting to a location that does not look like a URL", + ) from e + + scheme = parsed_redirect_url.scheme + if scheme not in HTTP_AND_EMPTY_SCHEMA_SET: + if req._body is not None: + await req._body.close() + resp.close() + raise NonHttpUrlRedirectClientError(r_url) + elif not scheme: + parsed_redirect_url = url.join(parsed_redirect_url) + + try: + redirect_origin = parsed_redirect_url.origin() + except ValueError as origin_val_err: + if req._body is not None: + await req._body.close() + resp.close() + raise InvalidUrlRedirectClientError( + parsed_redirect_url, + "Invalid redirect URL origin", + ) from origin_val_err + + if url.origin() != redirect_origin: + auth = None + headers.pop(hdrs.AUTHORIZATION, None) + + url = parsed_redirect_url + params = {} + resp.release() + continue + + break + + if req._body is not None: + await req._body.close() + # check response status + if raise_for_status is None: + raise_for_status = self._raise_for_status + + if raise_for_status is None: + pass + elif callable(raise_for_status): + await raise_for_status(resp) + elif raise_for_status: + resp.raise_for_status() + + # register connection + if handle is not None: + if resp.connection is not None: + resp.connection.add_callback(handle.cancel) + else: + handle.cancel() + + resp._history = tuple(history) + + for trace in traces: + await trace.send_request_end( + method, url.update_query(params), headers, resp + ) + return resp + + except BaseException as e: + # cleanup timer + tm.close() + if handle: + handle.cancel() + handle = None + + for trace in traces: + await trace.send_request_exception( + method, url.update_query(params), headers, e + ) + raise + + def ws_connect( + self, + url: StrOrURL, + *, + method: str = hdrs.METH_GET, + protocols: Iterable[str] = (), + timeout: Union[ClientWSTimeout, _SENTINEL] = sentinel, + receive_timeout: Optional[float] = None, + autoclose: bool = True, + autoping: bool = True, + heartbeat: Optional[float] = None, + auth: Optional[BasicAuth] = None, + origin: Optional[str] = None, + params: Query = None, + headers: Optional[LooseHeaders] = None, + proxy: Optional[StrOrURL] = None, + proxy_auth: Optional[BasicAuth] = None, + ssl: Union[SSLContext, bool, Fingerprint] = True, + verify_ssl: Optional[bool] = None, + fingerprint: Optional[bytes] = None, + ssl_context: Optional[SSLContext] = None, + server_hostname: Optional[str] = None, + proxy_headers: Optional[LooseHeaders] = None, + compress: int = 0, + max_msg_size: int = 4 * 1024 * 1024, + ) -> "_WSRequestContextManager": + """Initiate websocket connection.""" + return _WSRequestContextManager( + self._ws_connect( + url, + method=method, + protocols=protocols, + timeout=timeout, + receive_timeout=receive_timeout, + autoclose=autoclose, + autoping=autoping, + heartbeat=heartbeat, + auth=auth, + origin=origin, + params=params, + headers=headers, + proxy=proxy, + proxy_auth=proxy_auth, + ssl=ssl, + verify_ssl=verify_ssl, + fingerprint=fingerprint, + ssl_context=ssl_context, + server_hostname=server_hostname, + proxy_headers=proxy_headers, + compress=compress, + max_msg_size=max_msg_size, + ) + ) + + async def _ws_connect( + self, + url: StrOrURL, + *, + method: str = hdrs.METH_GET, + protocols: Iterable[str] = (), + timeout: Union[ClientWSTimeout, _SENTINEL] = sentinel, + receive_timeout: Optional[float] = None, + autoclose: bool = True, + autoping: bool = True, + heartbeat: Optional[float] = None, + auth: Optional[BasicAuth] = None, + origin: Optional[str] = None, + params: Query = None, + headers: Optional[LooseHeaders] = None, + proxy: Optional[StrOrURL] = None, + proxy_auth: Optional[BasicAuth] = None, + ssl: Union[SSLContext, bool, Fingerprint] = True, + verify_ssl: Optional[bool] = None, + fingerprint: Optional[bytes] = None, + ssl_context: Optional[SSLContext] = None, + server_hostname: Optional[str] = None, + proxy_headers: Optional[LooseHeaders] = None, + compress: int = 0, + max_msg_size: int = 4 * 1024 * 1024, + ) -> ClientWebSocketResponse: + if timeout is not sentinel: + if isinstance(timeout, ClientWSTimeout): + ws_timeout = timeout + else: + warnings.warn( + "parameter 'timeout' of type 'float' " + "is deprecated, please use " + "'timeout=ClientWSTimeout(ws_close=...)'", + DeprecationWarning, + stacklevel=2, + ) + ws_timeout = ClientWSTimeout(ws_close=timeout) + else: + ws_timeout = DEFAULT_WS_CLIENT_TIMEOUT + if receive_timeout is not None: + warnings.warn( + "float parameter 'receive_timeout' " + "is deprecated, please use parameter " + "'timeout=ClientWSTimeout(ws_receive=...)'", + DeprecationWarning, + stacklevel=2, + ) + ws_timeout = attr.evolve(ws_timeout, ws_receive=receive_timeout) + + if headers is None: + real_headers: CIMultiDict[str] = CIMultiDict() + else: + real_headers = CIMultiDict(headers) + + default_headers = { + hdrs.UPGRADE: "websocket", + hdrs.CONNECTION: "Upgrade", + hdrs.SEC_WEBSOCKET_VERSION: "13", + } + + for key, value in default_headers.items(): + real_headers.setdefault(key, value) + + sec_key = base64.b64encode(os.urandom(16)) + real_headers[hdrs.SEC_WEBSOCKET_KEY] = sec_key.decode() + + if protocols: + real_headers[hdrs.SEC_WEBSOCKET_PROTOCOL] = ",".join(protocols) + if origin is not None: + real_headers[hdrs.ORIGIN] = origin + if compress: + extstr = ws_ext_gen(compress=compress) + real_headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = extstr + + # For the sake of backward compatibility, if user passes in None, convert it to True + if ssl is None: + warnings.warn( + "ssl=None is deprecated, please use ssl=True", + DeprecationWarning, + stacklevel=2, + ) + ssl = True + ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint) + + # send request + resp = await self.request( + method, + url, + params=params, + headers=real_headers, + read_until_eof=False, + auth=auth, + proxy=proxy, + proxy_auth=proxy_auth, + ssl=ssl, + server_hostname=server_hostname, + proxy_headers=proxy_headers, + ) + + try: + # check handshake + if resp.status != 101: + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message="Invalid response status", + status=resp.status, + headers=resp.headers, + ) + + if resp.headers.get(hdrs.UPGRADE, "").lower() != "websocket": + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message="Invalid upgrade header", + status=resp.status, + headers=resp.headers, + ) + + if resp.headers.get(hdrs.CONNECTION, "").lower() != "upgrade": + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message="Invalid connection header", + status=resp.status, + headers=resp.headers, + ) + + # key calculation + r_key = resp.headers.get(hdrs.SEC_WEBSOCKET_ACCEPT, "") + match = base64.b64encode(hashlib.sha1(sec_key + WS_KEY).digest()).decode() + if r_key != match: + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message="Invalid challenge response", + status=resp.status, + headers=resp.headers, + ) + + # websocket protocol + protocol = None + if protocols and hdrs.SEC_WEBSOCKET_PROTOCOL in resp.headers: + resp_protocols = [ + proto.strip() + for proto in resp.headers[hdrs.SEC_WEBSOCKET_PROTOCOL].split(",") + ] + + for proto in resp_protocols: + if proto in protocols: + protocol = proto + break + + # websocket compress + notakeover = False + if compress: + compress_hdrs = resp.headers.get(hdrs.SEC_WEBSOCKET_EXTENSIONS) + if compress_hdrs: + try: + compress, notakeover = ws_ext_parse(compress_hdrs) + except WSHandshakeError as exc: + raise WSServerHandshakeError( + resp.request_info, + resp.history, + message=exc.args[0], + status=resp.status, + headers=resp.headers, + ) from exc + else: + compress = 0 + notakeover = False + + conn = resp.connection + assert conn is not None + conn_proto = conn.protocol + assert conn_proto is not None + + # For WS connection the read_timeout must be either receive_timeout or greater + # None == no timeout, i.e. infinite timeout, so None is the max timeout possible + if ws_timeout.ws_receive is None: + # Reset regardless + conn_proto.read_timeout = None + elif conn_proto.read_timeout is not None: + conn_proto.read_timeout = max( + ws_timeout.ws_receive, conn_proto.read_timeout + ) + + transport = conn.transport + assert transport is not None + reader = WebSocketDataQueue(conn_proto, 2**16, loop=self._loop) + conn_proto.set_parser(WebSocketReader(reader, max_msg_size), reader) + writer = WebSocketWriter( + conn_proto, + transport, + use_mask=True, + compress=compress, + notakeover=notakeover, + ) + except BaseException: + resp.close() + raise + else: + return self._ws_response_class( + reader, + writer, + protocol, + resp, + ws_timeout, + autoclose, + autoping, + self._loop, + heartbeat=heartbeat, + compress=compress, + client_notakeover=notakeover, + ) + + def _prepare_headers(self, headers: Optional[LooseHeaders]) -> "CIMultiDict[str]": + """Add default headers and transform it to CIMultiDict""" + # Convert headers to MultiDict + result = CIMultiDict(self._default_headers) + if headers: + if not isinstance(headers, (MultiDictProxy, MultiDict)): + headers = CIMultiDict(headers) + added_names: Set[str] = set() + for key, value in headers.items(): + if key in added_names: + result.add(key, value) + else: + result[key] = value + added_names.add(key) + return result + + if sys.version_info >= (3, 11) and TYPE_CHECKING: + + def get( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def options( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def head( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def post( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def put( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def patch( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + def delete( + self, + url: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> "_RequestContextManager": ... + + else: + + def get( + self, url: StrOrURL, *, allow_redirects: bool = True, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP GET request.""" + return _RequestContextManager( + self._request( + hdrs.METH_GET, url, allow_redirects=allow_redirects, **kwargs + ) + ) + + def options( + self, url: StrOrURL, *, allow_redirects: bool = True, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP OPTIONS request.""" + return _RequestContextManager( + self._request( + hdrs.METH_OPTIONS, url, allow_redirects=allow_redirects, **kwargs + ) + ) + + def head( + self, url: StrOrURL, *, allow_redirects: bool = False, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP HEAD request.""" + return _RequestContextManager( + self._request( + hdrs.METH_HEAD, url, allow_redirects=allow_redirects, **kwargs + ) + ) + + def post( + self, url: StrOrURL, *, data: Any = None, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP POST request.""" + return _RequestContextManager( + self._request(hdrs.METH_POST, url, data=data, **kwargs) + ) + + def put( + self, url: StrOrURL, *, data: Any = None, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP PUT request.""" + return _RequestContextManager( + self._request(hdrs.METH_PUT, url, data=data, **kwargs) + ) + + def patch( + self, url: StrOrURL, *, data: Any = None, **kwargs: Any + ) -> "_RequestContextManager": + """Perform HTTP PATCH request.""" + return _RequestContextManager( + self._request(hdrs.METH_PATCH, url, data=data, **kwargs) + ) + + def delete(self, url: StrOrURL, **kwargs: Any) -> "_RequestContextManager": + """Perform HTTP DELETE request.""" + return _RequestContextManager( + self._request(hdrs.METH_DELETE, url, **kwargs) + ) + + async def close(self) -> None: + """Close underlying connector. + + Release all acquired resources. + """ + if not self.closed: + if self._connector is not None and self._connector_owner: + await self._connector.close() + self._connector = None + + @property + def closed(self) -> bool: + """Is client session closed. + + A readonly property. + """ + return self._connector is None or self._connector.closed + + @property + def connector(self) -> Optional[BaseConnector]: + """Connector instance used for the session.""" + return self._connector + + @property + def cookie_jar(self) -> AbstractCookieJar: + """The session cookies.""" + return self._cookie_jar + + @property + def version(self) -> Tuple[int, int]: + """The session HTTP protocol version.""" + return self._version + + @property + def requote_redirect_url(self) -> bool: + """Do URL requoting on redirection handling.""" + return self._requote_redirect_url + + @requote_redirect_url.setter + def requote_redirect_url(self, val: bool) -> None: + """Do URL requoting on redirection handling.""" + warnings.warn( + "session.requote_redirect_url modification is deprecated #2778", + DeprecationWarning, + stacklevel=2, + ) + self._requote_redirect_url = val + + @property + def loop(self) -> asyncio.AbstractEventLoop: + """Session's loop.""" + warnings.warn( + "client.loop property is deprecated", DeprecationWarning, stacklevel=2 + ) + return self._loop + + @property + def timeout(self) -> ClientTimeout: + """Timeout for the session.""" + return self._timeout + + @property + def headers(self) -> "CIMultiDict[str]": + """The default headers of the client session.""" + return self._default_headers + + @property + def skip_auto_headers(self) -> FrozenSet[istr]: + """Headers for which autogeneration should be skipped""" + return self._skip_auto_headers + + @property + def auth(self) -> Optional[BasicAuth]: + """An object that represents HTTP Basic Authorization""" + return self._default_auth + + @property + def json_serialize(self) -> JSONEncoder: + """Json serializer callable""" + return self._json_serialize + + @property + def connector_owner(self) -> bool: + """Should connector be closed on session closing""" + return self._connector_owner + + @property + def raise_for_status( + self, + ) -> Union[bool, Callable[[ClientResponse], Awaitable[None]]]: + """Should `ClientResponse.raise_for_status()` be called for each response.""" + return self._raise_for_status + + @property + def auto_decompress(self) -> bool: + """Should the body response be automatically decompressed.""" + return self._auto_decompress + + @property + def trust_env(self) -> bool: + """ + Should proxies information from environment or netrc be trusted. + + Information is from HTTP_PROXY / HTTPS_PROXY environment variables + or ~/.netrc file if present. + """ + return self._trust_env + + @property + def trace_configs(self) -> List[TraceConfig]: + """A list of TraceConfig instances used for client tracing""" + return self._trace_configs + + def detach(self) -> None: + """Detach connector from session without closing the former. + + Session is switched to closed state anyway. + """ + self._connector = None + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + # __exit__ should exist in pair with __enter__ but never executed + pass # pragma: no cover + + async def __aenter__(self) -> "ClientSession": + return self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + await self.close() + + +class _BaseRequestContextManager(Coroutine[Any, Any, _RetType], Generic[_RetType]): + + __slots__ = ("_coro", "_resp") + + def __init__(self, coro: Coroutine["asyncio.Future[Any]", None, _RetType]) -> None: + self._coro: Coroutine["asyncio.Future[Any]", None, _RetType] = coro + + def send(self, arg: None) -> "asyncio.Future[Any]": + return self._coro.send(arg) + + def throw(self, *args: Any, **kwargs: Any) -> "asyncio.Future[Any]": + return self._coro.throw(*args, **kwargs) + + def close(self) -> None: + return self._coro.close() + + def __await__(self) -> Generator[Any, None, _RetType]: + ret = self._coro.__await__() + return ret + + def __iter__(self) -> Generator[Any, None, _RetType]: + return self.__await__() + + async def __aenter__(self) -> _RetType: + self._resp: _RetType = await self._coro + return await self._resp.__aenter__() + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc: Optional[BaseException], + tb: Optional[TracebackType], + ) -> None: + await self._resp.__aexit__(exc_type, exc, tb) + + +_RequestContextManager = _BaseRequestContextManager[ClientResponse] +_WSRequestContextManager = _BaseRequestContextManager[ClientWebSocketResponse] + + +class _SessionRequestContextManager: + + __slots__ = ("_coro", "_resp", "_session") + + def __init__( + self, + coro: Coroutine["asyncio.Future[Any]", None, ClientResponse], + session: ClientSession, + ) -> None: + self._coro = coro + self._resp: Optional[ClientResponse] = None + self._session = session + + async def __aenter__(self) -> ClientResponse: + try: + self._resp = await self._coro + except BaseException: + await self._session.close() + raise + else: + return self._resp + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc: Optional[BaseException], + tb: Optional[TracebackType], + ) -> None: + assert self._resp is not None + self._resp.close() + await self._session.close() + + +if sys.version_info >= (3, 11) and TYPE_CHECKING: + + def request( + method: str, + url: StrOrURL, + *, + version: HttpVersion = http.HttpVersion11, + connector: Optional[BaseConnector] = None, + loop: Optional[asyncio.AbstractEventLoop] = None, + **kwargs: Unpack[_RequestOptions], + ) -> _SessionRequestContextManager: ... + +else: + + def request( + method: str, + url: StrOrURL, + *, + version: HttpVersion = http.HttpVersion11, + connector: Optional[BaseConnector] = None, + loop: Optional[asyncio.AbstractEventLoop] = None, + **kwargs: Any, + ) -> _SessionRequestContextManager: + """Constructs and sends a request. + + Returns response object. + method - HTTP method + url - request url + params - (optional) Dictionary or bytes to be sent in the query + string of the new request + data - (optional) Dictionary, bytes, or file-like object to + send in the body of the request + json - (optional) Any json compatible python object + headers - (optional) Dictionary of HTTP Headers to send with + the request + cookies - (optional) Dict object to send with the request + auth - (optional) BasicAuth named tuple represent HTTP Basic Auth + auth - aiohttp.helpers.BasicAuth + allow_redirects - (optional) If set to False, do not follow + redirects + version - Request HTTP version. + compress - Set to True if request has to be compressed + with deflate encoding. + chunked - Set to chunk size for chunked transfer encoding. + expect100 - Expect 100-continue response from server. + connector - BaseConnector sub-class instance to support + connection pooling. + read_until_eof - Read response until eof if response + does not have Content-Length header. + loop - Optional event loop. + timeout - Optional ClientTimeout settings structure, 5min + total timeout by default. + Usage:: + >>> import aiohttp + >>> async with aiohttp.request('GET', 'http://python.org/') as resp: + ... print(resp) + ... data = await resp.read() + + """ + connector_owner = False + if connector is None: + connector_owner = True + connector = TCPConnector(loop=loop, force_close=True) + + session = ClientSession( + loop=loop, + cookies=kwargs.pop("cookies", None), + version=version, + timeout=kwargs.pop("timeout", sentinel), + connector=connector, + connector_owner=connector_owner, + ) + + return _SessionRequestContextManager( + session._request(method, url, **kwargs), + session, + ) diff --git a/lib/python3.1/site-packages/aiohttp/client_exceptions.py b/lib/python3.1/site-packages/aiohttp/client_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..1d298e9a8cf663cdc8a85d3b7d1f9264ff5e03c9 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/client_exceptions.py @@ -0,0 +1,421 @@ +"""HTTP related errors.""" + +import asyncio +import warnings +from typing import TYPE_CHECKING, Optional, Tuple, Union + +from multidict import MultiMapping + +from .typedefs import StrOrURL + +if TYPE_CHECKING: + import ssl + + SSLContext = ssl.SSLContext +else: + try: + import ssl + + SSLContext = ssl.SSLContext + except ImportError: # pragma: no cover + ssl = SSLContext = None # type: ignore[assignment] + +if TYPE_CHECKING: + from .client_reqrep import ClientResponse, ConnectionKey, Fingerprint, RequestInfo + from .http_parser import RawResponseMessage +else: + RequestInfo = ClientResponse = ConnectionKey = RawResponseMessage = None + +__all__ = ( + "ClientError", + "ClientConnectionError", + "ClientConnectionResetError", + "ClientOSError", + "ClientConnectorError", + "ClientProxyConnectionError", + "ClientSSLError", + "ClientConnectorDNSError", + "ClientConnectorSSLError", + "ClientConnectorCertificateError", + "ConnectionTimeoutError", + "SocketTimeoutError", + "ServerConnectionError", + "ServerTimeoutError", + "ServerDisconnectedError", + "ServerFingerprintMismatch", + "ClientResponseError", + "ClientHttpProxyError", + "WSServerHandshakeError", + "ContentTypeError", + "ClientPayloadError", + "InvalidURL", + "InvalidUrlClientError", + "RedirectClientError", + "NonHttpUrlClientError", + "InvalidUrlRedirectClientError", + "NonHttpUrlRedirectClientError", + "WSMessageTypeError", +) + + +class ClientError(Exception): + """Base class for client connection errors.""" + + +class ClientResponseError(ClientError): + """Base class for exceptions that occur after getting a response. + + request_info: An instance of RequestInfo. + history: A sequence of responses, if redirects occurred. + status: HTTP status code. + message: Error message. + headers: Response headers. + """ + + def __init__( + self, + request_info: RequestInfo, + history: Tuple[ClientResponse, ...], + *, + code: Optional[int] = None, + status: Optional[int] = None, + message: str = "", + headers: Optional[MultiMapping[str]] = None, + ) -> None: + self.request_info = request_info + if code is not None: + if status is not None: + raise ValueError( + "Both code and status arguments are provided; " + "code is deprecated, use status instead" + ) + warnings.warn( + "code argument is deprecated, use status instead", + DeprecationWarning, + stacklevel=2, + ) + if status is not None: + self.status = status + elif code is not None: + self.status = code + else: + self.status = 0 + self.message = message + self.headers = headers + self.history = history + self.args = (request_info, history) + + def __str__(self) -> str: + return "{}, message={!r}, url={!r}".format( + self.status, + self.message, + str(self.request_info.real_url), + ) + + def __repr__(self) -> str: + args = f"{self.request_info!r}, {self.history!r}" + if self.status != 0: + args += f", status={self.status!r}" + if self.message != "": + args += f", message={self.message!r}" + if self.headers is not None: + args += f", headers={self.headers!r}" + return f"{type(self).__name__}({args})" + + @property + def code(self) -> int: + warnings.warn( + "code property is deprecated, use status instead", + DeprecationWarning, + stacklevel=2, + ) + return self.status + + @code.setter + def code(self, value: int) -> None: + warnings.warn( + "code property is deprecated, use status instead", + DeprecationWarning, + stacklevel=2, + ) + self.status = value + + +class ContentTypeError(ClientResponseError): + """ContentType found is not valid.""" + + +class WSServerHandshakeError(ClientResponseError): + """websocket server handshake error.""" + + +class ClientHttpProxyError(ClientResponseError): + """HTTP proxy error. + + Raised in :class:`aiohttp.connector.TCPConnector` if + proxy responds with status other than ``200 OK`` + on ``CONNECT`` request. + """ + + +class TooManyRedirects(ClientResponseError): + """Client was redirected too many times.""" + + +class ClientConnectionError(ClientError): + """Base class for client socket errors.""" + + +class ClientConnectionResetError(ClientConnectionError, ConnectionResetError): + """ConnectionResetError""" + + +class ClientOSError(ClientConnectionError, OSError): + """OSError error.""" + + +class ClientConnectorError(ClientOSError): + """Client connector error. + + Raised in :class:`aiohttp.connector.TCPConnector` if + a connection can not be established. + """ + + def __init__(self, connection_key: ConnectionKey, os_error: OSError) -> None: + self._conn_key = connection_key + self._os_error = os_error + super().__init__(os_error.errno, os_error.strerror) + self.args = (connection_key, os_error) + + @property + def os_error(self) -> OSError: + return self._os_error + + @property + def host(self) -> str: + return self._conn_key.host + + @property + def port(self) -> Optional[int]: + return self._conn_key.port + + @property + def ssl(self) -> Union[SSLContext, bool, "Fingerprint"]: + return self._conn_key.ssl + + def __str__(self) -> str: + return "Cannot connect to host {0.host}:{0.port} ssl:{1} [{2}]".format( + self, "default" if self.ssl is True else self.ssl, self.strerror + ) + + # OSError.__reduce__ does too much black magick + __reduce__ = BaseException.__reduce__ + + +class ClientConnectorDNSError(ClientConnectorError): + """DNS resolution failed during client connection. + + Raised in :class:`aiohttp.connector.TCPConnector` if + DNS resolution fails. + """ + + +class ClientProxyConnectionError(ClientConnectorError): + """Proxy connection error. + + Raised in :class:`aiohttp.connector.TCPConnector` if + connection to proxy can not be established. + """ + + +class UnixClientConnectorError(ClientConnectorError): + """Unix connector error. + + Raised in :py:class:`aiohttp.connector.UnixConnector` + if connection to unix socket can not be established. + """ + + def __init__( + self, path: str, connection_key: ConnectionKey, os_error: OSError + ) -> None: + self._path = path + super().__init__(connection_key, os_error) + + @property + def path(self) -> str: + return self._path + + def __str__(self) -> str: + return "Cannot connect to unix socket {0.path} ssl:{1} [{2}]".format( + self, "default" if self.ssl is True else self.ssl, self.strerror + ) + + +class ServerConnectionError(ClientConnectionError): + """Server connection errors.""" + + +class ServerDisconnectedError(ServerConnectionError): + """Server disconnected.""" + + def __init__(self, message: Union[RawResponseMessage, str, None] = None) -> None: + if message is None: + message = "Server disconnected" + + self.args = (message,) + self.message = message + + +class ServerTimeoutError(ServerConnectionError, asyncio.TimeoutError): + """Server timeout error.""" + + +class ConnectionTimeoutError(ServerTimeoutError): + """Connection timeout error.""" + + +class SocketTimeoutError(ServerTimeoutError): + """Socket timeout error.""" + + +class ServerFingerprintMismatch(ServerConnectionError): + """SSL certificate does not match expected fingerprint.""" + + def __init__(self, expected: bytes, got: bytes, host: str, port: int) -> None: + self.expected = expected + self.got = got + self.host = host + self.port = port + self.args = (expected, got, host, port) + + def __repr__(self) -> str: + return "<{} expected={!r} got={!r} host={!r} port={!r}>".format( + self.__class__.__name__, self.expected, self.got, self.host, self.port + ) + + +class ClientPayloadError(ClientError): + """Response payload error.""" + + +class InvalidURL(ClientError, ValueError): + """Invalid URL. + + URL used for fetching is malformed, e.g. it doesn't contains host + part. + """ + + # Derive from ValueError for backward compatibility + + def __init__(self, url: StrOrURL, description: Union[str, None] = None) -> None: + # The type of url is not yarl.URL because the exception can be raised + # on URL(url) call + self._url = url + self._description = description + + if description: + super().__init__(url, description) + else: + super().__init__(url) + + @property + def url(self) -> StrOrURL: + return self._url + + @property + def description(self) -> "str | None": + return self._description + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {self}>" + + def __str__(self) -> str: + if self._description: + return f"{self._url} - {self._description}" + return str(self._url) + + +class InvalidUrlClientError(InvalidURL): + """Invalid URL client error.""" + + +class RedirectClientError(ClientError): + """Client redirect error.""" + + +class NonHttpUrlClientError(ClientError): + """Non http URL client error.""" + + +class InvalidUrlRedirectClientError(InvalidUrlClientError, RedirectClientError): + """Invalid URL redirect client error.""" + + +class NonHttpUrlRedirectClientError(NonHttpUrlClientError, RedirectClientError): + """Non http URL redirect client error.""" + + +class ClientSSLError(ClientConnectorError): + """Base error for ssl.*Errors.""" + + +if ssl is not None: + cert_errors = (ssl.CertificateError,) + cert_errors_bases = ( + ClientSSLError, + ssl.CertificateError, + ) + + ssl_errors = (ssl.SSLError,) + ssl_error_bases = (ClientSSLError, ssl.SSLError) +else: # pragma: no cover + cert_errors = tuple() + cert_errors_bases = ( + ClientSSLError, + ValueError, + ) + + ssl_errors = tuple() + ssl_error_bases = (ClientSSLError,) + + +class ClientConnectorSSLError(*ssl_error_bases): # type: ignore[misc] + """Response ssl error.""" + + +class ClientConnectorCertificateError(*cert_errors_bases): # type: ignore[misc] + """Response certificate error.""" + + def __init__( + self, connection_key: ConnectionKey, certificate_error: Exception + ) -> None: + self._conn_key = connection_key + self._certificate_error = certificate_error + self.args = (connection_key, certificate_error) + + @property + def certificate_error(self) -> Exception: + return self._certificate_error + + @property + def host(self) -> str: + return self._conn_key.host + + @property + def port(self) -> Optional[int]: + return self._conn_key.port + + @property + def ssl(self) -> bool: + return self._conn_key.is_ssl + + def __str__(self) -> str: + return ( + "Cannot connect to host {0.host}:{0.port} ssl:{0.ssl} " + "[{0.certificate_error.__class__.__name__}: " + "{0.certificate_error.args}]".format(self) + ) + + +class WSMessageTypeError(TypeError): + """WebSocket message type is not valid.""" diff --git a/lib/python3.1/site-packages/aiohttp/client_middleware_digest_auth.py b/lib/python3.1/site-packages/aiohttp/client_middleware_digest_auth.py new file mode 100644 index 0000000000000000000000000000000000000000..c1ed7ca0fdd9cad73a6bb03e4977e727f0591d90 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/client_middleware_digest_auth.py @@ -0,0 +1,476 @@ +""" +Digest authentication middleware for aiohttp client. + +This middleware implements HTTP Digest Authentication according to RFC 7616, +providing a more secure alternative to Basic Authentication. It supports all +standard hash algorithms including MD5, SHA, SHA-256, SHA-512 and their session +variants, as well as both 'auth' and 'auth-int' quality of protection (qop) options. +""" + +import hashlib +import os +import re +import time +from typing import ( + Callable, + Dict, + Final, + FrozenSet, + List, + Literal, + Tuple, + TypedDict, + Union, +) + +from yarl import URL + +from . import hdrs +from .client_exceptions import ClientError +from .client_middlewares import ClientHandlerType +from .client_reqrep import ClientRequest, ClientResponse +from .payload import Payload + + +class DigestAuthChallenge(TypedDict, total=False): + realm: str + nonce: str + qop: str + algorithm: str + opaque: str + domain: str + stale: str + + +DigestFunctions: Dict[str, Callable[[bytes], "hashlib._Hash"]] = { + "MD5": hashlib.md5, + "MD5-SESS": hashlib.md5, + "SHA": hashlib.sha1, + "SHA-SESS": hashlib.sha1, + "SHA256": hashlib.sha256, + "SHA256-SESS": hashlib.sha256, + "SHA-256": hashlib.sha256, + "SHA-256-SESS": hashlib.sha256, + "SHA512": hashlib.sha512, + "SHA512-SESS": hashlib.sha512, + "SHA-512": hashlib.sha512, + "SHA-512-SESS": hashlib.sha512, +} + + +# Compile the regex pattern once at module level for performance +_HEADER_PAIRS_PATTERN = re.compile( + r'(\w+)\s*=\s*(?:"((?:[^"\\]|\\.)*)"|([^\s,]+))' + # | | | | | | | | | || | + # +----|--|-|-|--|----|------|----|--||-----|--> alphanumeric key + # +--|-|-|--|----|------|----|--||-----|--> maybe whitespace + # | | | | | | | || | + # +-|-|--|----|------|----|--||-----|--> = (delimiter) + # +-|--|----|------|----|--||-----|--> maybe whitespace + # | | | | | || | + # +--|----|------|----|--||-----|--> group quoted or unquoted + # | | | | || | + # +----|------|----|--||-----|--> if quoted... + # +------|----|--||-----|--> anything but " or \ + # +----|--||-----|--> escaped characters allowed + # +--||-----|--> or can be empty string + # || | + # +|-----|--> if unquoted... + # +-----|--> anything but , or + # +--> at least one char req'd +) + + +# RFC 7616: Challenge parameters to extract +CHALLENGE_FIELDS: Final[ + Tuple[ + Literal["realm", "nonce", "qop", "algorithm", "opaque", "domain", "stale"], ... + ] +] = ( + "realm", + "nonce", + "qop", + "algorithm", + "opaque", + "domain", + "stale", +) + +# Supported digest authentication algorithms +# Use a tuple of sorted keys for predictable documentation and error messages +SUPPORTED_ALGORITHMS: Final[Tuple[str, ...]] = tuple(sorted(DigestFunctions.keys())) + +# RFC 7616: Fields that require quoting in the Digest auth header +# These fields must be enclosed in double quotes in the Authorization header. +# Algorithm, qop, and nc are never quoted per RFC specifications. +# This frozen set is used by the template-based header construction to +# automatically determine which fields need quotes. +QUOTED_AUTH_FIELDS: Final[FrozenSet[str]] = frozenset( + {"username", "realm", "nonce", "uri", "response", "opaque", "cnonce"} +) + + +def escape_quotes(value: str) -> str: + """Escape double quotes for HTTP header values.""" + return value.replace('"', '\\"') + + +def unescape_quotes(value: str) -> str: + """Unescape double quotes in HTTP header values.""" + return value.replace('\\"', '"') + + +def parse_header_pairs(header: str) -> Dict[str, str]: + """ + Parse key-value pairs from WWW-Authenticate or similar HTTP headers. + + This function handles the complex format of WWW-Authenticate header values, + supporting both quoted and unquoted values, proper handling of commas in + quoted values, and whitespace variations per RFC 7616. + + Examples of supported formats: + - key1="value1", key2=value2 + - key1 = "value1" , key2="value, with, commas" + - key1=value1,key2="value2" + - realm="example.com", nonce="12345", qop="auth" + + Args: + header: The header value string to parse + + Returns: + Dictionary mapping parameter names to their values + """ + return { + stripped_key: unescape_quotes(quoted_val) if quoted_val else unquoted_val + for key, quoted_val, unquoted_val in _HEADER_PAIRS_PATTERN.findall(header) + if (stripped_key := key.strip()) + } + + +class DigestAuthMiddleware: + """ + HTTP digest authentication middleware for aiohttp client. + + This middleware intercepts 401 Unauthorized responses containing a Digest + authentication challenge, calculates the appropriate digest credentials, + and automatically retries the request with the proper Authorization header. + + Features: + - Handles all aspects of Digest authentication handshake automatically + - Supports all standard hash algorithms: + - MD5, MD5-SESS + - SHA, SHA-SESS + - SHA256, SHA256-SESS, SHA-256, SHA-256-SESS + - SHA512, SHA512-SESS, SHA-512, SHA-512-SESS + - Supports 'auth' and 'auth-int' quality of protection modes + - Properly handles quoted strings and parameter parsing + - Includes replay attack protection with client nonce count tracking + - Supports preemptive authentication per RFC 7616 Section 3.6 + + Standards compliance: + - RFC 7616: HTTP Digest Access Authentication (primary reference) + - RFC 2617: HTTP Authentication (deprecated by RFC 7616) + - RFC 1945: Section 11.1 (username restrictions) + + Implementation notes: + The core digest calculation is inspired by the implementation in + https://github.com/requests/requests/blob/v2.18.4/requests/auth.py + with added support for modern digest auth features and error handling. + """ + + def __init__( + self, + login: str, + password: str, + preemptive: bool = True, + ) -> None: + if login is None: + raise ValueError("None is not allowed as login value") + + if password is None: + raise ValueError("None is not allowed as password value") + + if ":" in login: + raise ValueError('A ":" is not allowed in username (RFC 1945#section-11.1)') + + self._login_str: Final[str] = login + self._login_bytes: Final[bytes] = login.encode("utf-8") + self._password_bytes: Final[bytes] = password.encode("utf-8") + + self._last_nonce_bytes = b"" + self._nonce_count = 0 + self._challenge: DigestAuthChallenge = {} + self._preemptive: bool = preemptive + # Set of URLs defining the protection space + self._protection_space: List[str] = [] + + async def _encode( + self, method: str, url: URL, body: Union[Payload, Literal[b""]] + ) -> str: + """ + Build digest authorization header for the current challenge. + + Args: + method: The HTTP method (GET, POST, etc.) + url: The request URL + body: The request body (used for qop=auth-int) + + Returns: + A fully formatted Digest authorization header string + + Raises: + ClientError: If the challenge is missing required parameters or + contains unsupported values + + """ + challenge = self._challenge + if "realm" not in challenge: + raise ClientError( + "Malformed Digest auth challenge: Missing 'realm' parameter" + ) + + if "nonce" not in challenge: + raise ClientError( + "Malformed Digest auth challenge: Missing 'nonce' parameter" + ) + + # Empty realm values are allowed per RFC 7616 (SHOULD, not MUST, contain host name) + realm = challenge["realm"] + nonce = challenge["nonce"] + + # Empty nonce values are not allowed as they are security-critical for replay protection + if not nonce: + raise ClientError( + "Security issue: Digest auth challenge contains empty 'nonce' value" + ) + + qop_raw = challenge.get("qop", "") + # Preserve original algorithm case for response while using uppercase for processing + algorithm_original = challenge.get("algorithm", "MD5") + algorithm = algorithm_original.upper() + opaque = challenge.get("opaque", "") + + # Convert string values to bytes once + nonce_bytes = nonce.encode("utf-8") + realm_bytes = realm.encode("utf-8") + path = URL(url).path_qs + + # Process QoP + qop = "" + qop_bytes = b"" + if qop_raw: + valid_qops = {"auth", "auth-int"}.intersection( + {q.strip() for q in qop_raw.split(",") if q.strip()} + ) + if not valid_qops: + raise ClientError( + f"Digest auth error: Unsupported Quality of Protection (qop) value(s): {qop_raw}" + ) + + qop = "auth-int" if "auth-int" in valid_qops else "auth" + qop_bytes = qop.encode("utf-8") + + if algorithm not in DigestFunctions: + raise ClientError( + f"Digest auth error: Unsupported hash algorithm: {algorithm}. " + f"Supported algorithms: {', '.join(SUPPORTED_ALGORITHMS)}" + ) + hash_fn: Final = DigestFunctions[algorithm] + + def H(x: bytes) -> bytes: + """RFC 7616 Section 3: Hash function H(data) = hex(hash(data)).""" + return hash_fn(x).hexdigest().encode() + + def KD(s: bytes, d: bytes) -> bytes: + """RFC 7616 Section 3: KD(secret, data) = H(concat(secret, ":", data)).""" + return H(b":".join((s, d))) + + # Calculate A1 and A2 + A1 = b":".join((self._login_bytes, realm_bytes, self._password_bytes)) + A2 = f"{method.upper()}:{path}".encode() + if qop == "auth-int": + if isinstance(body, Payload): # will always be empty bytes unless Payload + entity_bytes = await body.as_bytes() # Get bytes from Payload + else: + entity_bytes = body + entity_hash = H(entity_bytes) + A2 = b":".join((A2, entity_hash)) + + HA1 = H(A1) + HA2 = H(A2) + + # Nonce count handling + if nonce_bytes == self._last_nonce_bytes: + self._nonce_count += 1 + else: + self._nonce_count = 1 + + self._last_nonce_bytes = nonce_bytes + ncvalue = f"{self._nonce_count:08x}" + ncvalue_bytes = ncvalue.encode("utf-8") + + # Generate client nonce + cnonce = hashlib.sha1( + b"".join( + [ + str(self._nonce_count).encode("utf-8"), + nonce_bytes, + time.ctime().encode("utf-8"), + os.urandom(8), + ] + ) + ).hexdigest()[:16] + cnonce_bytes = cnonce.encode("utf-8") + + # Special handling for session-based algorithms + if algorithm.upper().endswith("-SESS"): + HA1 = H(b":".join((HA1, nonce_bytes, cnonce_bytes))) + + # Calculate the response digest + if qop: + noncebit = b":".join( + (nonce_bytes, ncvalue_bytes, cnonce_bytes, qop_bytes, HA2) + ) + response_digest = KD(HA1, noncebit) + else: + response_digest = KD(HA1, b":".join((nonce_bytes, HA2))) + + # Define a dict mapping of header fields to their values + # Group fields into always-present, optional, and qop-dependent + header_fields = { + # Always present fields + "username": escape_quotes(self._login_str), + "realm": escape_quotes(realm), + "nonce": escape_quotes(nonce), + "uri": path, + "response": response_digest.decode(), + "algorithm": algorithm_original, + } + + # Optional fields + if opaque: + header_fields["opaque"] = escape_quotes(opaque) + + # QoP-dependent fields + if qop: + header_fields["qop"] = qop + header_fields["nc"] = ncvalue + header_fields["cnonce"] = cnonce + + # Build header using templates for each field type + pairs: List[str] = [] + for field, value in header_fields.items(): + if field in QUOTED_AUTH_FIELDS: + pairs.append(f'{field}="{value}"') + else: + pairs.append(f"{field}={value}") + + return f"Digest {', '.join(pairs)}" + + def _in_protection_space(self, url: URL) -> bool: + """ + Check if the given URL is within the current protection space. + + According to RFC 7616, a URI is in the protection space if any URI + in the protection space is a prefix of it (after both have been made absolute). + """ + request_str = str(url) + for space_str in self._protection_space: + # Check if request starts with space URL + if not request_str.startswith(space_str): + continue + # Exact match or space ends with / (proper directory prefix) + if len(request_str) == len(space_str) or space_str[-1] == "/": + return True + # Check next char is / to ensure proper path boundary + if request_str[len(space_str)] == "/": + return True + return False + + def _authenticate(self, response: ClientResponse) -> bool: + """ + Takes the given response and tries digest-auth, if needed. + + Returns true if the original request must be resent. + """ + if response.status != 401: + return False + + auth_header = response.headers.get("www-authenticate", "") + if not auth_header: + return False # No authentication header present + + method, sep, headers = auth_header.partition(" ") + if not sep: + # No space found in www-authenticate header + return False # Malformed auth header, missing scheme separator + + if method.lower() != "digest": + # Not a digest auth challenge (could be Basic, Bearer, etc.) + return False + + if not headers: + # We have a digest scheme but no parameters + return False # Malformed digest header, missing parameters + + # We have a digest auth header with content + if not (header_pairs := parse_header_pairs(headers)): + # Failed to parse any key-value pairs + return False # Malformed digest header, no valid parameters + + # Extract challenge parameters + self._challenge = {} + for field in CHALLENGE_FIELDS: + if value := header_pairs.get(field): + self._challenge[field] = value + + # Update protection space based on domain parameter or default to origin + origin = response.url.origin() + + if domain := self._challenge.get("domain"): + # Parse space-separated list of URIs + self._protection_space = [] + for uri in domain.split(): + # Remove quotes if present + uri = uri.strip('"') + if uri.startswith("/"): + # Path-absolute, relative to origin + self._protection_space.append(str(origin.join(URL(uri)))) + else: + # Absolute URI + self._protection_space.append(str(URL(uri))) + else: + # No domain specified, protection space is entire origin + self._protection_space = [str(origin)] + + # Return True only if we found at least one challenge parameter + return bool(self._challenge) + + async def __call__( + self, request: ClientRequest, handler: ClientHandlerType + ) -> ClientResponse: + """Run the digest auth middleware.""" + response = None + for retry_count in range(2): + # Apply authorization header if: + # 1. This is a retry after 401 (retry_count > 0), OR + # 2. Preemptive auth is enabled AND we have a challenge AND the URL is in protection space + if retry_count > 0 or ( + self._preemptive + and self._challenge + and self._in_protection_space(request.url) + ): + request.headers[hdrs.AUTHORIZATION] = await self._encode( + request.method, request.url, request.body + ) + + # Send the request + response = await handler(request) + + # Check if we need to authenticate + if not self._authenticate(response): + break + + # At this point, response is guaranteed to be defined + assert response is not None + return response diff --git a/lib/python3.1/site-packages/aiohttp/client_middlewares.py b/lib/python3.1/site-packages/aiohttp/client_middlewares.py new file mode 100644 index 0000000000000000000000000000000000000000..3ca2cb202ad93963369f2a10fd1d118a194c4405 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/client_middlewares.py @@ -0,0 +1,55 @@ +"""Client middleware support.""" + +from collections.abc import Awaitable, Callable, Sequence + +from .client_reqrep import ClientRequest, ClientResponse + +__all__ = ("ClientMiddlewareType", "ClientHandlerType", "build_client_middlewares") + +# Type alias for client request handlers - functions that process requests and return responses +ClientHandlerType = Callable[[ClientRequest], Awaitable[ClientResponse]] + +# Type for client middleware - similar to server but uses ClientRequest/ClientResponse +ClientMiddlewareType = Callable[ + [ClientRequest, ClientHandlerType], Awaitable[ClientResponse] +] + + +def build_client_middlewares( + handler: ClientHandlerType, + middlewares: Sequence[ClientMiddlewareType], +) -> ClientHandlerType: + """ + Apply middlewares to request handler. + + The middlewares are applied in reverse order, so the first middleware + in the list wraps all subsequent middlewares and the handler. + + This implementation avoids using partial/update_wrapper to minimize overhead + and doesn't cache to avoid holding references to stateful middleware. + """ + # Optimize for single middleware case + if len(middlewares) == 1: + middleware = middlewares[0] + + async def single_middleware_handler(req: ClientRequest) -> ClientResponse: + return await middleware(req, handler) + + return single_middleware_handler + + # Build the chain for multiple middlewares + current_handler = handler + + for middleware in reversed(middlewares): + # Create a new closure that captures the current state + def make_wrapper( + mw: ClientMiddlewareType, next_h: ClientHandlerType + ) -> ClientHandlerType: + async def wrapped(req: ClientRequest) -> ClientResponse: + return await mw(req, next_h) + + return wrapped + + current_handler = make_wrapper(middleware, current_handler) + + return current_handler diff --git a/lib/python3.1/site-packages/aiohttp/client_proto.py b/lib/python3.1/site-packages/aiohttp/client_proto.py new file mode 100644 index 0000000000000000000000000000000000000000..e2fb1ce64cb6a39f5a72aecfd5840536defba519 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/client_proto.py @@ -0,0 +1,359 @@ +import asyncio +from contextlib import suppress +from typing import Any, Optional, Tuple, Union + +from .base_protocol import BaseProtocol +from .client_exceptions import ( + ClientConnectionError, + ClientOSError, + ClientPayloadError, + ServerDisconnectedError, + SocketTimeoutError, +) +from .helpers import ( + _EXC_SENTINEL, + EMPTY_BODY_STATUS_CODES, + BaseTimerContext, + set_exception, + set_result, +) +from .http import HttpResponseParser, RawResponseMessage +from .http_exceptions import HttpProcessingError +from .streams import EMPTY_PAYLOAD, DataQueue, StreamReader + + +class ResponseHandler(BaseProtocol, DataQueue[Tuple[RawResponseMessage, StreamReader]]): + """Helper class to adapt between Protocol and StreamReader.""" + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + BaseProtocol.__init__(self, loop=loop) + DataQueue.__init__(self, loop) + + self._should_close = False + + self._payload: Optional[StreamReader] = None + self._skip_payload = False + self._payload_parser = None + + self._timer = None + + self._tail = b"" + self._upgraded = False + self._parser: Optional[HttpResponseParser] = None + + self._read_timeout: Optional[float] = None + self._read_timeout_handle: Optional[asyncio.TimerHandle] = None + + self._timeout_ceil_threshold: Optional[float] = 5 + + self._closed: Union[None, asyncio.Future[None]] = None + self._connection_lost_called = False + + @property + def closed(self) -> Union[None, asyncio.Future[None]]: + """Future that is set when the connection is closed. + + This property returns a Future that will be completed when the connection + is closed. The Future is created lazily on first access to avoid creating + futures that will never be awaited. + + Returns: + - A Future[None] if the connection is still open or was closed after + this property was accessed + - None if connection_lost() was already called before this property + was ever accessed (indicating no one is waiting for the closure) + """ + if self._closed is None and not self._connection_lost_called: + self._closed = self._loop.create_future() + return self._closed + + @property + def upgraded(self) -> bool: + return self._upgraded + + @property + def should_close(self) -> bool: + return bool( + self._should_close + or (self._payload is not None and not self._payload.is_eof()) + or self._upgraded + or self._exception is not None + or self._payload_parser is not None + or self._buffer + or self._tail + ) + + def force_close(self) -> None: + self._should_close = True + + def close(self) -> None: + self._exception = None # Break cyclic references + transport = self.transport + if transport is not None: + transport.close() + self.transport = None + self._payload = None + self._drop_timeout() + + def abort(self) -> None: + self._exception = None # Break cyclic references + transport = self.transport + if transport is not None: + transport.abort() + self.transport = None + self._payload = None + self._drop_timeout() + + def is_connected(self) -> bool: + return self.transport is not None and not self.transport.is_closing() + + def connection_lost(self, exc: Optional[BaseException]) -> None: + self._connection_lost_called = True + self._drop_timeout() + + original_connection_error = exc + reraised_exc = original_connection_error + + connection_closed_cleanly = original_connection_error is None + + if self._closed is not None: + # If someone is waiting for the closed future, + # we should set it to None or an exception. If + # self._closed is None, it means that + # connection_lost() was called already + # or nobody is waiting for it. + if connection_closed_cleanly: + set_result(self._closed, None) + else: + assert original_connection_error is not None + set_exception( + self._closed, + ClientConnectionError( + f"Connection lost: {original_connection_error !s}", + ), + original_connection_error, + ) + + if self._payload_parser is not None: + with suppress(Exception): # FIXME: log this somehow? + self._payload_parser.feed_eof() + + uncompleted = None + if self._parser is not None: + try: + uncompleted = self._parser.feed_eof() + except Exception as underlying_exc: + if self._payload is not None: + client_payload_exc_msg = ( + f"Response payload is not completed: {underlying_exc !r}" + ) + if not connection_closed_cleanly: + client_payload_exc_msg = ( + f"{client_payload_exc_msg !s}. " + f"{original_connection_error !r}" + ) + set_exception( + self._payload, + ClientPayloadError(client_payload_exc_msg), + underlying_exc, + ) + + if not self.is_eof(): + if isinstance(original_connection_error, OSError): + reraised_exc = ClientOSError(*original_connection_error.args) + if connection_closed_cleanly: + reraised_exc = ServerDisconnectedError(uncompleted) + # assigns self._should_close to True as side effect, + # we do it anyway below + underlying_non_eof_exc = ( + _EXC_SENTINEL + if connection_closed_cleanly + else original_connection_error + ) + assert underlying_non_eof_exc is not None + assert reraised_exc is not None + self.set_exception(reraised_exc, underlying_non_eof_exc) + + self._should_close = True + self._parser = None + self._payload = None + self._payload_parser = None + self._reading_paused = False + + super().connection_lost(reraised_exc) + + def eof_received(self) -> None: + # should call parser.feed_eof() most likely + self._drop_timeout() + + def pause_reading(self) -> None: + super().pause_reading() + self._drop_timeout() + + def resume_reading(self) -> None: + super().resume_reading() + self._reschedule_timeout() + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + self._should_close = True + self._drop_timeout() + super().set_exception(exc, exc_cause) + + def set_parser(self, parser: Any, payload: Any) -> None: + # TODO: actual types are: + # parser: WebSocketReader + # payload: WebSocketDataQueue + # but they are not generi enough + # Need an ABC for both types + self._payload = payload + self._payload_parser = parser + + self._drop_timeout() + + if self._tail: + data, self._tail = self._tail, b"" + self.data_received(data) + + def set_response_params( + self, + *, + timer: Optional[BaseTimerContext] = None, + skip_payload: bool = False, + read_until_eof: bool = False, + auto_decompress: bool = True, + read_timeout: Optional[float] = None, + read_bufsize: int = 2**16, + timeout_ceil_threshold: float = 5, + max_line_size: int = 8190, + max_field_size: int = 8190, + ) -> None: + self._skip_payload = skip_payload + + self._read_timeout = read_timeout + + self._timeout_ceil_threshold = timeout_ceil_threshold + + self._parser = HttpResponseParser( + self, + self._loop, + read_bufsize, + timer=timer, + payload_exception=ClientPayloadError, + response_with_body=not skip_payload, + read_until_eof=read_until_eof, + auto_decompress=auto_decompress, + max_line_size=max_line_size, + max_field_size=max_field_size, + ) + + if self._tail: + data, self._tail = self._tail, b"" + self.data_received(data) + + def _drop_timeout(self) -> None: + if self._read_timeout_handle is not None: + self._read_timeout_handle.cancel() + self._read_timeout_handle = None + + def _reschedule_timeout(self) -> None: + timeout = self._read_timeout + if self._read_timeout_handle is not None: + self._read_timeout_handle.cancel() + + if timeout: + self._read_timeout_handle = self._loop.call_later( + timeout, self._on_read_timeout + ) + else: + self._read_timeout_handle = None + + def start_timeout(self) -> None: + self._reschedule_timeout() + + @property + def read_timeout(self) -> Optional[float]: + return self._read_timeout + + @read_timeout.setter + def read_timeout(self, read_timeout: Optional[float]) -> None: + self._read_timeout = read_timeout + + def _on_read_timeout(self) -> None: + exc = SocketTimeoutError("Timeout on reading data from socket") + self.set_exception(exc) + if self._payload is not None: + set_exception(self._payload, exc) + + def data_received(self, data: bytes) -> None: + self._reschedule_timeout() + + if not data: + return + + # custom payload parser - currently always WebSocketReader + if self._payload_parser is not None: + eof, tail = self._payload_parser.feed_data(data) + if eof: + self._payload = None + self._payload_parser = None + + if tail: + self.data_received(tail) + return + + if self._upgraded or self._parser is None: + # i.e. websocket connection, websocket parser is not set yet + self._tail += data + return + + # parse http messages + try: + messages, upgraded, tail = self._parser.feed_data(data) + except BaseException as underlying_exc: + if self.transport is not None: + # connection.release() could be called BEFORE + # data_received(), the transport is already + # closed in this case + self.transport.close() + # should_close is True after the call + if isinstance(underlying_exc, HttpProcessingError): + exc = HttpProcessingError( + code=underlying_exc.code, + message=underlying_exc.message, + headers=underlying_exc.headers, + ) + else: + exc = HttpProcessingError() + self.set_exception(exc, underlying_exc) + return + + self._upgraded = upgraded + + payload: Optional[StreamReader] = None + for message, payload in messages: + if message.should_close: + self._should_close = True + + self._payload = payload + + if self._skip_payload or message.code in EMPTY_BODY_STATUS_CODES: + self.feed_data((message, EMPTY_PAYLOAD), 0) + else: + self.feed_data((message, payload), 0) + + if payload is not None: + # new message(s) was processed + # register timeout handler unsubscribing + # either on end-of-stream or immediately for + # EMPTY_PAYLOAD + if payload is not EMPTY_PAYLOAD: + payload.on_eof(self._drop_timeout) + else: + self._drop_timeout() + + if upgraded and tail: + self.data_received(tail) diff --git a/lib/python3.1/site-packages/aiohttp/client_reqrep.py b/lib/python3.1/site-packages/aiohttp/client_reqrep.py new file mode 100644 index 0000000000000000000000000000000000000000..2586119b288b27b3500ef08d7051d2667c146a26 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/client_reqrep.py @@ -0,0 +1,1533 @@ +import asyncio +import codecs +import contextlib +import functools +import io +import re +import sys +import traceback +import warnings +from collections.abc import Mapping +from hashlib import md5, sha1, sha256 +from http.cookies import Morsel, SimpleCookie +from types import MappingProxyType, TracebackType +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Literal, + NamedTuple, + Optional, + Tuple, + Type, + Union, +) + +import attr +from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy +from yarl import URL + +from . import hdrs, helpers, http, multipart, payload +from ._cookie_helpers import ( + parse_cookie_header, + parse_set_cookie_headers, + preserve_morsel_with_coded_value, +) +from .abc import AbstractStreamWriter +from .client_exceptions import ( + ClientConnectionError, + ClientOSError, + ClientResponseError, + ContentTypeError, + InvalidURL, + ServerFingerprintMismatch, +) +from .compression_utils import HAS_BROTLI +from .formdata import FormData +from .helpers import ( + _SENTINEL, + BaseTimerContext, + BasicAuth, + HeadersMixin, + TimerNoop, + basicauth_from_netrc, + netrc_from_env, + noop, + reify, + set_exception, + set_result, +) +from .http import ( + SERVER_SOFTWARE, + HttpVersion, + HttpVersion10, + HttpVersion11, + StreamWriter, +) +from .streams import StreamReader +from .typedefs import ( + DEFAULT_JSON_DECODER, + JSONDecoder, + LooseCookies, + LooseHeaders, + Query, + RawHeaders, +) + +if TYPE_CHECKING: + import ssl + from ssl import SSLContext +else: + try: + import ssl + from ssl import SSLContext + except ImportError: # pragma: no cover + ssl = None # type: ignore[assignment] + SSLContext = object # type: ignore[misc,assignment] + + +__all__ = ("ClientRequest", "ClientResponse", "RequestInfo", "Fingerprint") + + +if TYPE_CHECKING: + from .client import ClientSession + from .connector import Connection + from .tracing import Trace + + +_CONNECTION_CLOSED_EXCEPTION = ClientConnectionError("Connection closed") +_CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") +json_re = re.compile(r"^application/(?:[\w.+-]+?\+)?json") + + +def _gen_default_accept_encoding() -> str: + return "gzip, deflate, br" if HAS_BROTLI else "gzip, deflate" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ContentDisposition: + type: Optional[str] + parameters: "MappingProxyType[str, str]" + filename: Optional[str] + + +class _RequestInfo(NamedTuple): + url: URL + method: str + headers: "CIMultiDictProxy[str]" + real_url: URL + + +class RequestInfo(_RequestInfo): + + def __new__( + cls, + url: URL, + method: str, + headers: "CIMultiDictProxy[str]", + real_url: URL = _SENTINEL, # type: ignore[assignment] + ) -> "RequestInfo": + """Create a new RequestInfo instance. + + For backwards compatibility, the real_url parameter is optional. + """ + return tuple.__new__( + cls, (url, method, headers, url if real_url is _SENTINEL else real_url) + ) + + +class Fingerprint: + HASHFUNC_BY_DIGESTLEN = { + 16: md5, + 20: sha1, + 32: sha256, + } + + def __init__(self, fingerprint: bytes) -> None: + digestlen = len(fingerprint) + hashfunc = self.HASHFUNC_BY_DIGESTLEN.get(digestlen) + if not hashfunc: + raise ValueError("fingerprint has invalid length") + elif hashfunc is md5 or hashfunc is sha1: + raise ValueError("md5 and sha1 are insecure and not supported. Use sha256.") + self._hashfunc = hashfunc + self._fingerprint = fingerprint + + @property + def fingerprint(self) -> bytes: + return self._fingerprint + + def check(self, transport: asyncio.Transport) -> None: + if not transport.get_extra_info("sslcontext"): + return + sslobj = transport.get_extra_info("ssl_object") + cert = sslobj.getpeercert(binary_form=True) + got = self._hashfunc(cert).digest() + if got != self._fingerprint: + host, port, *_ = transport.get_extra_info("peername") + raise ServerFingerprintMismatch(self._fingerprint, got, host, port) + + +if ssl is not None: + SSL_ALLOWED_TYPES = (ssl.SSLContext, bool, Fingerprint, type(None)) +else: # pragma: no cover + SSL_ALLOWED_TYPES = (bool, type(None)) + + +def _merge_ssl_params( + ssl: Union["SSLContext", bool, Fingerprint], + verify_ssl: Optional[bool], + ssl_context: Optional["SSLContext"], + fingerprint: Optional[bytes], +) -> Union["SSLContext", bool, Fingerprint]: + if ssl is None: + ssl = True # Double check for backwards compatibility + if verify_ssl is not None and not verify_ssl: + warnings.warn( + "verify_ssl is deprecated, use ssl=False instead", + DeprecationWarning, + stacklevel=3, + ) + if ssl is not True: + raise ValueError( + "verify_ssl, ssl_context, fingerprint and ssl " + "parameters are mutually exclusive" + ) + else: + ssl = False + if ssl_context is not None: + warnings.warn( + "ssl_context is deprecated, use ssl=context instead", + DeprecationWarning, + stacklevel=3, + ) + if ssl is not True: + raise ValueError( + "verify_ssl, ssl_context, fingerprint and ssl " + "parameters are mutually exclusive" + ) + else: + ssl = ssl_context + if fingerprint is not None: + warnings.warn( + "fingerprint is deprecated, use ssl=Fingerprint(fingerprint) instead", + DeprecationWarning, + stacklevel=3, + ) + if ssl is not True: + raise ValueError( + "verify_ssl, ssl_context, fingerprint and ssl " + "parameters are mutually exclusive" + ) + else: + ssl = Fingerprint(fingerprint) + if not isinstance(ssl, SSL_ALLOWED_TYPES): + raise TypeError( + "ssl should be SSLContext, bool, Fingerprint or None, " + "got {!r} instead.".format(ssl) + ) + return ssl + + +_SSL_SCHEMES = frozenset(("https", "wss")) + + +# ConnectionKey is a NamedTuple because it is used as a key in a dict +# and a set in the connector. Since a NamedTuple is a tuple it uses +# the fast native tuple __hash__ and __eq__ implementation in CPython. +class ConnectionKey(NamedTuple): + # the key should contain an information about used proxy / TLS + # to prevent reusing wrong connections from a pool + host: str + port: Optional[int] + is_ssl: bool + ssl: Union[SSLContext, bool, Fingerprint] + proxy: Optional[URL] + proxy_auth: Optional[BasicAuth] + proxy_headers_hash: Optional[int] # hash(CIMultiDict) + + +def _is_expected_content_type( + response_content_type: str, expected_content_type: str +) -> bool: + if expected_content_type == "application/json": + return json_re.match(response_content_type) is not None + return expected_content_type in response_content_type + + +def _warn_if_unclosed_payload(payload: payload.Payload, stacklevel: int = 2) -> None: + """Warn if the payload is not closed. + + Callers must check that the body is a Payload before calling this method. + + Args: + payload: The payload to check + stacklevel: Stack level for the warning (default 2 for direct callers) + """ + if not payload.autoclose and not payload.consumed: + warnings.warn( + "The previous request body contains unclosed resources. " + "Use await request.update_body() instead of setting request.body " + "directly to properly close resources and avoid leaks.", + ResourceWarning, + stacklevel=stacklevel, + ) + + +class ClientResponse(HeadersMixin): + + # Some of these attributes are None when created, + # but will be set by the start() method. + # As the end user will likely never see the None values, we cheat the types below. + # from the Status-Line of the response + version: Optional[HttpVersion] = None # HTTP-Version + status: int = None # type: ignore[assignment] # Status-Code + reason: Optional[str] = None # Reason-Phrase + + content: StreamReader = None # type: ignore[assignment] # Payload stream + _body: Optional[bytes] = None + _headers: CIMultiDictProxy[str] = None # type: ignore[assignment] + _history: Tuple["ClientResponse", ...] = () + _raw_headers: RawHeaders = None # type: ignore[assignment] + + _connection: Optional["Connection"] = None # current connection + _cookies: Optional[SimpleCookie] = None + _raw_cookie_headers: Optional[Tuple[str, ...]] = None + _continue: Optional["asyncio.Future[bool]"] = None + _source_traceback: Optional[traceback.StackSummary] = None + _session: Optional["ClientSession"] = None + # set up by ClientRequest after ClientResponse object creation + # post-init stage allows to not change ctor signature + _closed = True # to allow __del__ for non-initialized properly response + _released = False + _in_context = False + + _resolve_charset: Callable[["ClientResponse", bytes], str] = lambda *_: "utf-8" + + __writer: Optional["asyncio.Task[None]"] = None + + def __init__( + self, + method: str, + url: URL, + *, + writer: "Optional[asyncio.Task[None]]", + continue100: Optional["asyncio.Future[bool]"], + timer: BaseTimerContext, + request_info: RequestInfo, + traces: List["Trace"], + loop: asyncio.AbstractEventLoop, + session: "ClientSession", + ) -> None: + # URL forbids subclasses, so a simple type check is enough. + assert type(url) is URL + + self.method = method + + self._real_url = url + self._url = url.with_fragment(None) if url.raw_fragment else url + if writer is not None: + self._writer = writer + if continue100 is not None: + self._continue = continue100 + self._request_info = request_info + self._timer = timer if timer is not None else TimerNoop() + self._cache: Dict[str, Any] = {} + self._traces = traces + self._loop = loop + # Save reference to _resolve_charset, so that get_encoding() will still + # work after the response has finished reading the body. + # TODO: Fix session=None in tests (see ClientRequest.__init__). + if session is not None: + # store a reference to session #1985 + self._session = session + self._resolve_charset = session._resolve_charset + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + def __reset_writer(self, _: object = None) -> None: + self.__writer = None + + @property + def _writer(self) -> Optional["asyncio.Task[None]"]: + """The writer task for streaming data. + + _writer is only provided for backwards compatibility + for subclasses that may need to access it. + """ + return self.__writer + + @_writer.setter + def _writer(self, writer: Optional["asyncio.Task[None]"]) -> None: + """Set the writer task for streaming data.""" + if self.__writer is not None: + self.__writer.remove_done_callback(self.__reset_writer) + self.__writer = writer + if writer is None: + return + if writer.done(): + # The writer is already done, so we can clear it immediately. + self.__writer = None + else: + writer.add_done_callback(self.__reset_writer) + + @property + def cookies(self) -> SimpleCookie: + if self._cookies is None: + if self._raw_cookie_headers is not None: + # Parse cookies for response.cookies (SimpleCookie for backward compatibility) + cookies = SimpleCookie() + # Use parse_set_cookie_headers for more lenient parsing that handles + # malformed cookies better than SimpleCookie.load + cookies.update(parse_set_cookie_headers(self._raw_cookie_headers)) + self._cookies = cookies + else: + self._cookies = SimpleCookie() + return self._cookies + + @cookies.setter + def cookies(self, cookies: SimpleCookie) -> None: + self._cookies = cookies + # Generate raw cookie headers from the SimpleCookie + if cookies: + self._raw_cookie_headers = tuple( + morsel.OutputString() for morsel in cookies.values() + ) + else: + self._raw_cookie_headers = None + + @reify + def url(self) -> URL: + return self._url + + @reify + def url_obj(self) -> URL: + warnings.warn("Deprecated, use .url #1654", DeprecationWarning, stacklevel=2) + return self._url + + @reify + def real_url(self) -> URL: + return self._real_url + + @reify + def host(self) -> str: + assert self._url.host is not None + return self._url.host + + @reify + def headers(self) -> "CIMultiDictProxy[str]": + return self._headers + + @reify + def raw_headers(self) -> RawHeaders: + return self._raw_headers + + @reify + def request_info(self) -> RequestInfo: + return self._request_info + + @reify + def content_disposition(self) -> Optional[ContentDisposition]: + raw = self._headers.get(hdrs.CONTENT_DISPOSITION) + if raw is None: + return None + disposition_type, params_dct = multipart.parse_content_disposition(raw) + params = MappingProxyType(params_dct) + filename = multipart.content_disposition_filename(params) + return ContentDisposition(disposition_type, params, filename) + + def __del__(self, _warnings: Any = warnings) -> None: + if self._closed: + return + + if self._connection is not None: + self._connection.release() + self._cleanup_writer() + + if self._loop.get_debug(): + kwargs = {"source": self} + _warnings.warn(f"Unclosed response {self!r}", ResourceWarning, **kwargs) + context = {"client_response": self, "message": "Unclosed response"} + if self._source_traceback: + context["source_traceback"] = self._source_traceback + self._loop.call_exception_handler(context) + + def __repr__(self) -> str: + out = io.StringIO() + ascii_encodable_url = str(self.url) + if self.reason: + ascii_encodable_reason = self.reason.encode( + "ascii", "backslashreplace" + ).decode("ascii") + else: + ascii_encodable_reason = "None" + print( + "".format( + ascii_encodable_url, self.status, ascii_encodable_reason + ), + file=out, + ) + print(self.headers, file=out) + return out.getvalue() + + @property + def connection(self) -> Optional["Connection"]: + return self._connection + + @reify + def history(self) -> Tuple["ClientResponse", ...]: + """A sequence of of responses, if redirects occurred.""" + return self._history + + @reify + def links(self) -> "MultiDictProxy[MultiDictProxy[Union[str, URL]]]": + links_str = ", ".join(self.headers.getall("link", [])) + + if not links_str: + return MultiDictProxy(MultiDict()) + + links: MultiDict[MultiDictProxy[Union[str, URL]]] = MultiDict() + + for val in re.split(r",(?=\s*<)", links_str): + match = re.match(r"\s*<(.*)>(.*)", val) + if match is None: # pragma: no cover + # the check exists to suppress mypy error + continue + url, params_str = match.groups() + params = params_str.split(";")[1:] + + link: MultiDict[Union[str, URL]] = MultiDict() + + for param in params: + match = re.match(r"^\s*(\S*)\s*=\s*(['\"]?)(.*?)(\2)\s*$", param, re.M) + if match is None: # pragma: no cover + # the check exists to suppress mypy error + continue + key, _, value, _ = match.groups() + + link.add(key, value) + + key = link.get("rel", url) + + link.add("url", self.url.join(URL(url))) + + links.add(str(key), MultiDictProxy(link)) + + return MultiDictProxy(links) + + async def start(self, connection: "Connection") -> "ClientResponse": + """Start response processing.""" + self._closed = False + self._protocol = connection.protocol + self._connection = connection + + with self._timer: + while True: + # read response + try: + protocol = self._protocol + message, payload = await protocol.read() # type: ignore[union-attr] + except http.HttpProcessingError as exc: + raise ClientResponseError( + self.request_info, + self.history, + status=exc.code, + message=exc.message, + headers=exc.headers, + ) from exc + + if message.code < 100 or message.code > 199 or message.code == 101: + break + + if self._continue is not None: + set_result(self._continue, True) + self._continue = None + + # payload eof handler + payload.on_eof(self._response_eof) + + # response status + self.version = message.version + self.status = message.code + self.reason = message.reason + + # headers + self._headers = message.headers # type is CIMultiDictProxy + self._raw_headers = message.raw_headers # type is Tuple[bytes, bytes] + + # payload + self.content = payload + + # cookies + if cookie_hdrs := self.headers.getall(hdrs.SET_COOKIE, ()): + # Store raw cookie headers for CookieJar + self._raw_cookie_headers = tuple(cookie_hdrs) + return self + + def _response_eof(self) -> None: + if self._closed: + return + + # protocol could be None because connection could be detached + protocol = self._connection and self._connection.protocol + if protocol is not None and protocol.upgraded: + return + + self._closed = True + self._cleanup_writer() + self._release_connection() + + @property + def closed(self) -> bool: + return self._closed + + def close(self) -> None: + if not self._released: + self._notify_content() + + self._closed = True + if self._loop is None or self._loop.is_closed(): + return + + self._cleanup_writer() + if self._connection is not None: + self._connection.close() + self._connection = None + + def release(self) -> Any: + if not self._released: + self._notify_content() + + self._closed = True + + self._cleanup_writer() + self._release_connection() + return noop() + + @property + def ok(self) -> bool: + """Returns ``True`` if ``status`` is less than ``400``, ``False`` if not. + + This is **not** a check for ``200 OK`` but a check that the response + status is under 400. + """ + return 400 > self.status + + def raise_for_status(self) -> None: + if not self.ok: + # reason should always be not None for a started response + assert self.reason is not None + + # If we're in a context we can rely on __aexit__() to release as the + # exception propagates. + if not self._in_context: + self.release() + + raise ClientResponseError( + self.request_info, + self.history, + status=self.status, + message=self.reason, + headers=self.headers, + ) + + def _release_connection(self) -> None: + if self._connection is not None: + if self.__writer is None: + self._connection.release() + self._connection = None + else: + self.__writer.add_done_callback(lambda f: self._release_connection()) + + async def _wait_released(self) -> None: + if self.__writer is not None: + try: + await self.__writer + except asyncio.CancelledError: + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + self._release_connection() + + def _cleanup_writer(self) -> None: + if self.__writer is not None: + self.__writer.cancel() + self._session = None + + def _notify_content(self) -> None: + content = self.content + if content and content.exception() is None: + set_exception(content, _CONNECTION_CLOSED_EXCEPTION) + self._released = True + + async def wait_for_close(self) -> None: + if self.__writer is not None: + try: + await self.__writer + except asyncio.CancelledError: + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + self.release() + + async def read(self) -> bytes: + """Read response payload.""" + if self._body is None: + try: + self._body = await self.content.read() + for trace in self._traces: + await trace.send_response_chunk_received( + self.method, self.url, self._body + ) + except BaseException: + self.close() + raise + elif self._released: # Response explicitly released + raise ClientConnectionError("Connection closed") + + protocol = self._connection and self._connection.protocol + if protocol is None or not protocol.upgraded: + await self._wait_released() # Underlying connection released + return self._body + + def get_encoding(self) -> str: + ctype = self.headers.get(hdrs.CONTENT_TYPE, "").lower() + mimetype = helpers.parse_mimetype(ctype) + + encoding = mimetype.parameters.get("charset") + if encoding: + with contextlib.suppress(LookupError, ValueError): + return codecs.lookup(encoding).name + + if mimetype.type == "application" and ( + mimetype.subtype == "json" or mimetype.subtype == "rdap" + ): + # RFC 7159 states that the default encoding is UTF-8. + # RFC 7483 defines application/rdap+json + return "utf-8" + + if self._body is None: + raise RuntimeError( + "Cannot compute fallback encoding of a not yet read body" + ) + + return self._resolve_charset(self, self._body) + + async def text(self, encoding: Optional[str] = None, errors: str = "strict") -> str: + """Read response payload and decode.""" + if self._body is None: + await self.read() + + if encoding is None: + encoding = self.get_encoding() + + return self._body.decode(encoding, errors=errors) # type: ignore[union-attr] + + async def json( + self, + *, + encoding: Optional[str] = None, + loads: JSONDecoder = DEFAULT_JSON_DECODER, + content_type: Optional[str] = "application/json", + ) -> Any: + """Read and decodes JSON response.""" + if self._body is None: + await self.read() + + if content_type: + ctype = self.headers.get(hdrs.CONTENT_TYPE, "").lower() + if not _is_expected_content_type(ctype, content_type): + raise ContentTypeError( + self.request_info, + self.history, + status=self.status, + message=( + "Attempt to decode JSON with unexpected mimetype: %s" % ctype + ), + headers=self.headers, + ) + + stripped = self._body.strip() # type: ignore[union-attr] + if not stripped: + return None + + if encoding is None: + encoding = self.get_encoding() + + return loads(stripped.decode(encoding)) + + async def __aenter__(self) -> "ClientResponse": + self._in_context = True + return self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self._in_context = False + # similar to _RequestContextManager, we do not need to check + # for exceptions, response object can close connection + # if state is broken + self.release() + await self.wait_for_close() + + +class ClientRequest: + GET_METHODS = { + hdrs.METH_GET, + hdrs.METH_HEAD, + hdrs.METH_OPTIONS, + hdrs.METH_TRACE, + } + POST_METHODS = {hdrs.METH_PATCH, hdrs.METH_POST, hdrs.METH_PUT} + ALL_METHODS = GET_METHODS.union(POST_METHODS).union({hdrs.METH_DELETE}) + + DEFAULT_HEADERS = { + hdrs.ACCEPT: "*/*", + hdrs.ACCEPT_ENCODING: _gen_default_accept_encoding(), + } + + # Type of body depends on PAYLOAD_REGISTRY, which is dynamic. + _body: Union[None, payload.Payload] = None + auth = None + response = None + + __writer: Optional["asyncio.Task[None]"] = None # async task for streaming data + + # These class defaults help create_autospec() work correctly. + # If autospec is improved in future, maybe these can be removed. + url = URL() + method = "GET" + + _continue = None # waiter future for '100 Continue' response + + _skip_auto_headers: Optional["CIMultiDict[None]"] = None + + # N.B. + # Adding __del__ method with self._writer closing doesn't make sense + # because _writer is instance method, thus it keeps a reference to self. + # Until writer has finished finalizer will not be called. + + def __init__( + self, + method: str, + url: URL, + *, + params: Query = None, + headers: Optional[LooseHeaders] = None, + skip_auto_headers: Optional[Iterable[str]] = None, + data: Any = None, + cookies: Optional[LooseCookies] = None, + auth: Optional[BasicAuth] = None, + version: http.HttpVersion = http.HttpVersion11, + compress: Union[str, bool, None] = None, + chunked: Optional[bool] = None, + expect100: bool = False, + loop: Optional[asyncio.AbstractEventLoop] = None, + response_class: Optional[Type["ClientResponse"]] = None, + proxy: Optional[URL] = None, + proxy_auth: Optional[BasicAuth] = None, + timer: Optional[BaseTimerContext] = None, + session: Optional["ClientSession"] = None, + ssl: Union[SSLContext, bool, Fingerprint] = True, + proxy_headers: Optional[LooseHeaders] = None, + traces: Optional[List["Trace"]] = None, + trust_env: bool = False, + server_hostname: Optional[str] = None, + ): + if loop is None: + loop = asyncio.get_event_loop() + if match := _CONTAINS_CONTROL_CHAR_RE.search(method): + raise ValueError( + f"Method cannot contain non-token characters {method!r} " + f"(found at least {match.group()!r})" + ) + # URL forbids subclasses, so a simple type check is enough. + assert type(url) is URL, url + if proxy is not None: + assert type(proxy) is URL, proxy + # FIXME: session is None in tests only, need to fix tests + # assert session is not None + if TYPE_CHECKING: + assert session is not None + self._session = session + if params: + url = url.extend_query(params) + self.original_url = url + self.url = url.with_fragment(None) if url.raw_fragment else url + self.method = method.upper() + self.chunked = chunked + self.compress = compress + self.loop = loop + self.length = None + if response_class is None: + real_response_class = ClientResponse + else: + real_response_class = response_class + self.response_class: Type[ClientResponse] = real_response_class + self._timer = timer if timer is not None else TimerNoop() + self._ssl = ssl if ssl is not None else True + self.server_hostname = server_hostname + + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + self.update_version(version) + self.update_host(url) + self.update_headers(headers) + self.update_auto_headers(skip_auto_headers) + self.update_cookies(cookies) + self.update_content_encoding(data) + self.update_auth(auth, trust_env) + self.update_proxy(proxy, proxy_auth, proxy_headers) + + self.update_body_from_data(data) + if data is not None or self.method not in self.GET_METHODS: + self.update_transfer_encoding() + self.update_expect_continue(expect100) + self._traces = [] if traces is None else traces + + def __reset_writer(self, _: object = None) -> None: + self.__writer = None + + def _get_content_length(self) -> Optional[int]: + """Extract and validate Content-Length header value. + + Returns parsed Content-Length value or None if not set. + Raises ValueError if header exists but cannot be parsed as an integer. + """ + if hdrs.CONTENT_LENGTH not in self.headers: + return None + + content_length_hdr = self.headers[hdrs.CONTENT_LENGTH] + try: + return int(content_length_hdr) + except ValueError: + raise ValueError( + f"Invalid Content-Length header: {content_length_hdr}" + ) from None + + @property + def skip_auto_headers(self) -> CIMultiDict[None]: + return self._skip_auto_headers or CIMultiDict() + + @property + def _writer(self) -> Optional["asyncio.Task[None]"]: + return self.__writer + + @_writer.setter + def _writer(self, writer: "asyncio.Task[None]") -> None: + if self.__writer is not None: + self.__writer.remove_done_callback(self.__reset_writer) + self.__writer = writer + writer.add_done_callback(self.__reset_writer) + + def is_ssl(self) -> bool: + return self.url.scheme in _SSL_SCHEMES + + @property + def ssl(self) -> Union["SSLContext", bool, Fingerprint]: + return self._ssl + + @property + def connection_key(self) -> ConnectionKey: + if proxy_headers := self.proxy_headers: + h: Optional[int] = hash(tuple(proxy_headers.items())) + else: + h = None + url = self.url + return tuple.__new__( + ConnectionKey, + ( + url.raw_host or "", + url.port, + url.scheme in _SSL_SCHEMES, + self._ssl, + self.proxy, + self.proxy_auth, + h, + ), + ) + + @property + def host(self) -> str: + ret = self.url.raw_host + assert ret is not None + return ret + + @property + def port(self) -> Optional[int]: + return self.url.port + + @property + def body(self) -> Union[payload.Payload, Literal[b""]]: + """Request body.""" + # empty body is represented as bytes for backwards compatibility + return self._body or b"" + + @body.setter + def body(self, value: Any) -> None: + """Set request body with warning for non-autoclose payloads. + + WARNING: This setter must be called from within an event loop and is not + thread-safe. Setting body outside of an event loop may raise RuntimeError + when closing file-based payloads. + + DEPRECATED: Direct assignment to body is deprecated and will be removed + in a future version. Use await update_body() instead for proper resource + management. + """ + # Close existing payload if present + if self._body is not None: + # Warn if the payload needs manual closing + # stacklevel=3: user code -> body setter -> _warn_if_unclosed_payload + _warn_if_unclosed_payload(self._body, stacklevel=3) + # NOTE: In the future, when we remove sync close support, + # this setter will need to be removed and only the async + # update_body() method will be available. For now, we call + # _close() for backwards compatibility. + self._body._close() + self._update_body(value) + + @property + def request_info(self) -> RequestInfo: + headers: CIMultiDictProxy[str] = CIMultiDictProxy(self.headers) + # These are created on every request, so we use a NamedTuple + # for performance reasons. We don't use the RequestInfo.__new__ + # method because it has a different signature which is provided + # for backwards compatibility only. + return tuple.__new__( + RequestInfo, (self.url, self.method, headers, self.original_url) + ) + + @property + def session(self) -> "ClientSession": + """Return the ClientSession instance. + + This property provides access to the ClientSession that initiated + this request, allowing middleware to make additional requests + using the same session. + """ + return self._session + + def update_host(self, url: URL) -> None: + """Update destination host, port and connection type (ssl).""" + # get host/port + if not url.raw_host: + raise InvalidURL(url) + + # basic auth info + if url.raw_user or url.raw_password: + self.auth = helpers.BasicAuth(url.user or "", url.password or "") + + def update_version(self, version: Union[http.HttpVersion, str]) -> None: + """Convert request version to two elements tuple. + + parser HTTP version '1.1' => (1, 1) + """ + if isinstance(version, str): + v = [part.strip() for part in version.split(".", 1)] + try: + version = http.HttpVersion(int(v[0]), int(v[1])) + except ValueError: + raise ValueError( + f"Can not parse http version number: {version}" + ) from None + self.version = version + + def update_headers(self, headers: Optional[LooseHeaders]) -> None: + """Update request headers.""" + self.headers: CIMultiDict[str] = CIMultiDict() + + # Build the host header + host = self.url.host_port_subcomponent + + # host_port_subcomponent is None when the URL is a relative URL. + # but we know we do not have a relative URL here. + assert host is not None + self.headers[hdrs.HOST] = host + + if not headers: + return + + if isinstance(headers, (dict, MultiDictProxy, MultiDict)): + headers = headers.items() + + for key, value in headers: # type: ignore[misc] + # A special case for Host header + if key in hdrs.HOST_ALL: + self.headers[key] = value + else: + self.headers.add(key, value) + + def update_auto_headers(self, skip_auto_headers: Optional[Iterable[str]]) -> None: + if skip_auto_headers is not None: + self._skip_auto_headers = CIMultiDict( + (hdr, None) for hdr in sorted(skip_auto_headers) + ) + used_headers = self.headers.copy() + used_headers.extend(self._skip_auto_headers) # type: ignore[arg-type] + else: + # Fast path when there are no headers to skip + # which is the most common case. + used_headers = self.headers + + for hdr, val in self.DEFAULT_HEADERS.items(): + if hdr not in used_headers: + self.headers[hdr] = val + + if hdrs.USER_AGENT not in used_headers: + self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE + + def update_cookies(self, cookies: Optional[LooseCookies]) -> None: + """Update request cookies header.""" + if not cookies: + return + + c = SimpleCookie() + if hdrs.COOKIE in self.headers: + # parse_cookie_header for RFC 6265 compliant Cookie header parsing + c.update(parse_cookie_header(self.headers.get(hdrs.COOKIE, ""))) + del self.headers[hdrs.COOKIE] + + if isinstance(cookies, Mapping): + iter_cookies = cookies.items() + else: + iter_cookies = cookies # type: ignore[assignment] + for name, value in iter_cookies: + if isinstance(value, Morsel): + # Use helper to preserve coded_value exactly as sent by server + c[name] = preserve_morsel_with_coded_value(value) + else: + c[name] = value # type: ignore[assignment] + + self.headers[hdrs.COOKIE] = c.output(header="", sep=";").strip() + + def update_content_encoding(self, data: Any) -> None: + """Set request content encoding.""" + if not data: + # Don't compress an empty body. + self.compress = None + return + + if self.headers.get(hdrs.CONTENT_ENCODING): + if self.compress: + raise ValueError( + "compress can not be set if Content-Encoding header is set" + ) + elif self.compress: + if not isinstance(self.compress, str): + self.compress = "deflate" + self.headers[hdrs.CONTENT_ENCODING] = self.compress + self.chunked = True # enable chunked, no need to deal with length + + def update_transfer_encoding(self) -> None: + """Analyze transfer-encoding header.""" + te = self.headers.get(hdrs.TRANSFER_ENCODING, "").lower() + + if "chunked" in te: + if self.chunked: + raise ValueError( + "chunked can not be set " + 'if "Transfer-Encoding: chunked" header is set' + ) + + elif self.chunked: + if hdrs.CONTENT_LENGTH in self.headers: + raise ValueError( + "chunked can not be set if Content-Length header is set" + ) + + self.headers[hdrs.TRANSFER_ENCODING] = "chunked" + + def update_auth(self, auth: Optional[BasicAuth], trust_env: bool = False) -> None: + """Set basic auth.""" + if auth is None: + auth = self.auth + if auth is None and trust_env and self.url.host is not None: + netrc_obj = netrc_from_env() + with contextlib.suppress(LookupError): + auth = basicauth_from_netrc(netrc_obj, self.url.host) + if auth is None: + return + + if not isinstance(auth, helpers.BasicAuth): + raise TypeError("BasicAuth() tuple is required instead") + + self.headers[hdrs.AUTHORIZATION] = auth.encode() + + def update_body_from_data(self, body: Any, _stacklevel: int = 3) -> None: + """Update request body from data.""" + if self._body is not None: + _warn_if_unclosed_payload(self._body, stacklevel=_stacklevel) + + if body is None: + self._body = None + # Set Content-Length to 0 when body is None for methods that expect a body + if ( + self.method not in self.GET_METHODS + and not self.chunked + and hdrs.CONTENT_LENGTH not in self.headers + ): + self.headers[hdrs.CONTENT_LENGTH] = "0" + return + + # FormData + maybe_payload = body() if isinstance(body, FormData) else body + + try: + body_payload = payload.PAYLOAD_REGISTRY.get(maybe_payload, disposition=None) + except payload.LookupError: + body_payload = FormData(maybe_payload)() # type: ignore[arg-type] + + self._body = body_payload + # enable chunked encoding if needed + if not self.chunked and hdrs.CONTENT_LENGTH not in self.headers: + if (size := body_payload.size) is not None: + self.headers[hdrs.CONTENT_LENGTH] = str(size) + else: + self.chunked = True + + # copy payload headers + assert body_payload.headers + headers = self.headers + skip_headers = self._skip_auto_headers + for key, value in body_payload.headers.items(): + if key in headers or (skip_headers is not None and key in skip_headers): + continue + headers[key] = value + + def _update_body(self, body: Any) -> None: + """Update request body after its already been set.""" + # Remove existing Content-Length header since body is changing + if hdrs.CONTENT_LENGTH in self.headers: + del self.headers[hdrs.CONTENT_LENGTH] + + # Remove existing Transfer-Encoding header to avoid conflicts + if self.chunked and hdrs.TRANSFER_ENCODING in self.headers: + del self.headers[hdrs.TRANSFER_ENCODING] + + # Now update the body using the existing method + # Called from _update_body, add 1 to stacklevel from caller + self.update_body_from_data(body, _stacklevel=4) + + # Update transfer encoding headers if needed (same logic as __init__) + if body is not None or self.method not in self.GET_METHODS: + self.update_transfer_encoding() + + async def update_body(self, body: Any) -> None: + """ + Update request body and close previous payload if needed. + + This method safely updates the request body by first closing any existing + payload to prevent resource leaks, then setting the new body. + + IMPORTANT: Always use this method instead of setting request.body directly. + Direct assignment to request.body will leak resources if the previous body + contains file handles, streams, or other resources that need cleanup. + + Args: + body: The new body content. Can be: + - bytes/bytearray: Raw binary data + - str: Text data (will be encoded using charset from Content-Type) + - FormData: Form data that will be encoded as multipart/form-data + - Payload: A pre-configured payload object + - AsyncIterable: An async iterable of bytes chunks + - File-like object: Will be read and sent as binary data + - None: Clears the body + + Usage: + # CORRECT: Use update_body + await request.update_body(b"new request data") + + # WRONG: Don't set body directly + # request.body = b"new request data" # This will leak resources! + + # Update with form data + form_data = FormData() + form_data.add_field('field', 'value') + await request.update_body(form_data) + + # Clear body + await request.update_body(None) + + Note: + This method is async because it may need to close file handles or + other resources associated with the previous payload. Always await + this method to ensure proper cleanup. + + Warning: + Setting request.body directly is highly discouraged and can lead to: + - Resource leaks (unclosed file handles, streams) + - Memory leaks (unreleased buffers) + - Unexpected behavior with streaming payloads + + It is not recommended to change the payload type in middleware. If the + body was already set (e.g., as bytes), it's best to keep the same type + rather than converting it (e.g., to str) as this may result in unexpected + behavior. + + See Also: + - update_body_from_data: Synchronous body update without cleanup + - body property: Direct body access (STRONGLY DISCOURAGED) + + """ + # Close existing payload if it exists and needs closing + if self._body is not None: + await self._body.close() + self._update_body(body) + + def update_expect_continue(self, expect: bool = False) -> None: + if expect: + self.headers[hdrs.EXPECT] = "100-continue" + elif ( + hdrs.EXPECT in self.headers + and self.headers[hdrs.EXPECT].lower() == "100-continue" + ): + expect = True + + if expect: + self._continue = self.loop.create_future() + + def update_proxy( + self, + proxy: Optional[URL], + proxy_auth: Optional[BasicAuth], + proxy_headers: Optional[LooseHeaders], + ) -> None: + self.proxy = proxy + if proxy is None: + self.proxy_auth = None + self.proxy_headers = None + return + + if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth): + raise ValueError("proxy_auth must be None or BasicAuth() tuple") + self.proxy_auth = proxy_auth + + if proxy_headers is not None and not isinstance( + proxy_headers, (MultiDict, MultiDictProxy) + ): + proxy_headers = CIMultiDict(proxy_headers) + self.proxy_headers = proxy_headers + + async def write_bytes( + self, + writer: AbstractStreamWriter, + conn: "Connection", + content_length: Optional[int] = None, + ) -> None: + """ + Write the request body to the connection stream. + + This method handles writing different types of request bodies: + 1. Payload objects (using their specialized write_with_length method) + 2. Bytes/bytearray objects + 3. Iterable body content + + Args: + writer: The stream writer to write the body to + conn: The connection being used for this request + content_length: Optional maximum number of bytes to write from the body + (None means write the entire body) + + The method properly handles: + - Waiting for 100-Continue responses if required + - Content length constraints for chunked encoding + - Error handling for network issues, cancellation, and other exceptions + - Signaling EOF and timeout management + + Raises: + ClientOSError: When there's an OS-level error writing the body + ClientConnectionError: When there's a general connection error + asyncio.CancelledError: When the operation is cancelled + + """ + # 100 response + if self._continue is not None: + # Force headers to be sent before waiting for 100-continue + writer.send_headers() + await writer.drain() + await self._continue + + protocol = conn.protocol + assert protocol is not None + try: + # This should be a rare case but the + # self._body can be set to None while + # the task is being started or we wait above + # for the 100-continue response. + # The more likely case is we have an empty + # payload, but 100-continue is still expected. + if self._body is not None: + await self._body.write_with_length(writer, content_length) + except OSError as underlying_exc: + reraised_exc = underlying_exc + + # Distinguish between timeout and other OS errors for better error reporting + exc_is_not_timeout = underlying_exc.errno is not None or not isinstance( + underlying_exc, asyncio.TimeoutError + ) + if exc_is_not_timeout: + reraised_exc = ClientOSError( + underlying_exc.errno, + f"Can not write request body for {self.url !s}", + ) + + set_exception(protocol, reraised_exc, underlying_exc) + except asyncio.CancelledError: + # Body hasn't been fully sent, so connection can't be reused + conn.close() + raise + except Exception as underlying_exc: + set_exception( + protocol, + ClientConnectionError( + "Failed to send bytes into the underlying connection " + f"{conn !s}: {underlying_exc!r}", + ), + underlying_exc, + ) + else: + # Successfully wrote the body, signal EOF and start response timeout + await writer.write_eof() + protocol.start_timeout() + + async def send(self, conn: "Connection") -> "ClientResponse": + # Specify request target: + # - CONNECT request must send authority form URI + # - not CONNECT proxy must send absolute form URI + # - most common is origin form URI + if self.method == hdrs.METH_CONNECT: + connect_host = self.url.host_subcomponent + assert connect_host is not None + path = f"{connect_host}:{self.url.port}" + elif self.proxy and not self.is_ssl(): + path = str(self.url) + else: + path = self.url.raw_path_qs + + protocol = conn.protocol + assert protocol is not None + writer = StreamWriter( + protocol, + self.loop, + on_chunk_sent=( + functools.partial(self._on_chunk_request_sent, self.method, self.url) + if self._traces + else None + ), + on_headers_sent=( + functools.partial(self._on_headers_request_sent, self.method, self.url) + if self._traces + else None + ), + ) + + if self.compress: + writer.enable_compression(self.compress) # type: ignore[arg-type] + + if self.chunked is not None: + writer.enable_chunking() + + # set default content-type + if ( + self.method in self.POST_METHODS + and ( + self._skip_auto_headers is None + or hdrs.CONTENT_TYPE not in self._skip_auto_headers + ) + and hdrs.CONTENT_TYPE not in self.headers + ): + self.headers[hdrs.CONTENT_TYPE] = "application/octet-stream" + + v = self.version + if hdrs.CONNECTION not in self.headers: + if conn._connector.force_close: + if v == HttpVersion11: + self.headers[hdrs.CONNECTION] = "close" + elif v == HttpVersion10: + self.headers[hdrs.CONNECTION] = "keep-alive" + + # status + headers + status_line = f"{self.method} {path} HTTP/{v.major}.{v.minor}" + + # Buffer headers for potential coalescing with body + await writer.write_headers(status_line, self.headers) + + task: Optional["asyncio.Task[None]"] + if self._body or self._continue is not None or protocol.writing_paused: + coro = self.write_bytes(writer, conn, self._get_content_length()) + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to write + # bytes immediately to avoid having to schedule + # the task on the event loop. + task = asyncio.Task(coro, loop=self.loop, eager_start=True) + else: + task = self.loop.create_task(coro) + if task.done(): + task = None + else: + self._writer = task + else: + # We have nothing to write because + # - there is no body + # - the protocol does not have writing paused + # - we are not waiting for a 100-continue response + protocol.start_timeout() + writer.set_eof() + task = None + response_class = self.response_class + assert response_class is not None + self.response = response_class( + self.method, + self.original_url, + writer=task, + continue100=self._continue, + timer=self._timer, + request_info=self.request_info, + traces=self._traces, + loop=self.loop, + session=self._session, + ) + return self.response + + async def close(self) -> None: + if self.__writer is not None: + try: + await self.__writer + except asyncio.CancelledError: + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + + def terminate(self) -> None: + if self.__writer is not None: + if not self.loop.is_closed(): + self.__writer.cancel() + self.__writer.remove_done_callback(self.__reset_writer) + self.__writer = None + + async def _on_chunk_request_sent(self, method: str, url: URL, chunk: bytes) -> None: + for trace in self._traces: + await trace.send_request_chunk_sent(method, url, chunk) + + async def _on_headers_request_sent( + self, method: str, url: URL, headers: "CIMultiDict[str]" + ) -> None: + for trace in self._traces: + await trace.send_request_headers(method, url, headers) diff --git a/lib/python3.1/site-packages/aiohttp/client_ws.py b/lib/python3.1/site-packages/aiohttp/client_ws.py new file mode 100644 index 0000000000000000000000000000000000000000..daa57d1930b1b0cfc03594aaae62bd93da5da165 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/client_ws.py @@ -0,0 +1,428 @@ +"""WebSocket client for asyncio.""" + +import asyncio +import sys +from types import TracebackType +from typing import Any, Optional, Type, cast + +import attr + +from ._websocket.reader import WebSocketDataQueue +from .client_exceptions import ClientError, ServerTimeoutError, WSMessageTypeError +from .client_reqrep import ClientResponse +from .helpers import calculate_timeout_when, set_result +from .http import ( + WS_CLOSED_MESSAGE, + WS_CLOSING_MESSAGE, + WebSocketError, + WSCloseCode, + WSMessage, + WSMsgType, +) +from .http_websocket import _INTERNAL_RECEIVE_TYPES, WebSocketWriter +from .streams import EofStream +from .typedefs import ( + DEFAULT_JSON_DECODER, + DEFAULT_JSON_ENCODER, + JSONDecoder, + JSONEncoder, +) + +if sys.version_info >= (3, 11): + import asyncio as async_timeout +else: + import async_timeout + + +@attr.s(frozen=True, slots=True) +class ClientWSTimeout: + ws_receive = attr.ib(type=Optional[float], default=None) + ws_close = attr.ib(type=Optional[float], default=None) + + +DEFAULT_WS_CLIENT_TIMEOUT = ClientWSTimeout(ws_receive=None, ws_close=10.0) + + +class ClientWebSocketResponse: + def __init__( + self, + reader: WebSocketDataQueue, + writer: WebSocketWriter, + protocol: Optional[str], + response: ClientResponse, + timeout: ClientWSTimeout, + autoclose: bool, + autoping: bool, + loop: asyncio.AbstractEventLoop, + *, + heartbeat: Optional[float] = None, + compress: int = 0, + client_notakeover: bool = False, + ) -> None: + self._response = response + self._conn = response.connection + + self._writer = writer + self._reader = reader + self._protocol = protocol + self._closed = False + self._closing = False + self._close_code: Optional[int] = None + self._timeout = timeout + self._autoclose = autoclose + self._autoping = autoping + self._heartbeat = heartbeat + self._heartbeat_cb: Optional[asyncio.TimerHandle] = None + self._heartbeat_when: float = 0.0 + if heartbeat is not None: + self._pong_heartbeat = heartbeat / 2.0 + self._pong_response_cb: Optional[asyncio.TimerHandle] = None + self._loop = loop + self._waiting: bool = False + self._close_wait: Optional[asyncio.Future[None]] = None + self._exception: Optional[BaseException] = None + self._compress = compress + self._client_notakeover = client_notakeover + self._ping_task: Optional[asyncio.Task[None]] = None + + self._reset_heartbeat() + + def _cancel_heartbeat(self) -> None: + self._cancel_pong_response_cb() + if self._heartbeat_cb is not None: + self._heartbeat_cb.cancel() + self._heartbeat_cb = None + if self._ping_task is not None: + self._ping_task.cancel() + self._ping_task = None + + def _cancel_pong_response_cb(self) -> None: + if self._pong_response_cb is not None: + self._pong_response_cb.cancel() + self._pong_response_cb = None + + def _reset_heartbeat(self) -> None: + if self._heartbeat is None: + return + self._cancel_pong_response_cb() + loop = self._loop + assert loop is not None + conn = self._conn + timeout_ceil_threshold = ( + conn._connector._timeout_ceil_threshold if conn is not None else 5 + ) + now = loop.time() + when = calculate_timeout_when(now, self._heartbeat, timeout_ceil_threshold) + self._heartbeat_when = when + if self._heartbeat_cb is None: + # We do not cancel the previous heartbeat_cb here because + # it generates a significant amount of TimerHandle churn + # which causes asyncio to rebuild the heap frequently. + # Instead _send_heartbeat() will reschedule the next + # heartbeat if it fires too early. + self._heartbeat_cb = loop.call_at(when, self._send_heartbeat) + + def _send_heartbeat(self) -> None: + self._heartbeat_cb = None + loop = self._loop + now = loop.time() + if now < self._heartbeat_when: + # Heartbeat fired too early, reschedule + self._heartbeat_cb = loop.call_at( + self._heartbeat_when, self._send_heartbeat + ) + return + + conn = self._conn + timeout_ceil_threshold = ( + conn._connector._timeout_ceil_threshold if conn is not None else 5 + ) + when = calculate_timeout_when(now, self._pong_heartbeat, timeout_ceil_threshold) + self._cancel_pong_response_cb() + self._pong_response_cb = loop.call_at(when, self._pong_not_received) + + coro = self._writer.send_frame(b"", WSMsgType.PING) + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to send the ping + # immediately to avoid having to schedule + # the task on the event loop. + ping_task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + ping_task = loop.create_task(coro) + + if not ping_task.done(): + self._ping_task = ping_task + ping_task.add_done_callback(self._ping_task_done) + else: + self._ping_task_done(ping_task) + + def _ping_task_done(self, task: "asyncio.Task[None]") -> None: + """Callback for when the ping task completes.""" + if not task.cancelled() and (exc := task.exception()): + self._handle_ping_pong_exception(exc) + self._ping_task = None + + def _pong_not_received(self) -> None: + self._handle_ping_pong_exception( + ServerTimeoutError(f"No PONG received after {self._pong_heartbeat} seconds") + ) + + def _handle_ping_pong_exception(self, exc: BaseException) -> None: + """Handle exceptions raised during ping/pong processing.""" + if self._closed: + return + self._set_closed() + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._exception = exc + self._response.close() + if self._waiting and not self._closing: + self._reader.feed_data(WSMessage(WSMsgType.ERROR, exc, None), 0) + + def _set_closed(self) -> None: + """Set the connection to closed. + + Cancel any heartbeat timers and set the closed flag. + """ + self._closed = True + self._cancel_heartbeat() + + def _set_closing(self) -> None: + """Set the connection to closing. + + Cancel any heartbeat timers and set the closing flag. + """ + self._closing = True + self._cancel_heartbeat() + + @property + def closed(self) -> bool: + return self._closed + + @property + def close_code(self) -> Optional[int]: + return self._close_code + + @property + def protocol(self) -> Optional[str]: + return self._protocol + + @property + def compress(self) -> int: + return self._compress + + @property + def client_notakeover(self) -> bool: + return self._client_notakeover + + def get_extra_info(self, name: str, default: Any = None) -> Any: + """extra info from connection transport""" + conn = self._response.connection + if conn is None: + return default + transport = conn.transport + if transport is None: + return default + return transport.get_extra_info(name, default) + + def exception(self) -> Optional[BaseException]: + return self._exception + + async def ping(self, message: bytes = b"") -> None: + await self._writer.send_frame(message, WSMsgType.PING) + + async def pong(self, message: bytes = b"") -> None: + await self._writer.send_frame(message, WSMsgType.PONG) + + async def send_frame( + self, message: bytes, opcode: WSMsgType, compress: Optional[int] = None + ) -> None: + """Send a frame over the websocket.""" + await self._writer.send_frame(message, opcode, compress) + + async def send_str(self, data: str, compress: Optional[int] = None) -> None: + if not isinstance(data, str): + raise TypeError("data argument must be str (%r)" % type(data)) + await self._writer.send_frame( + data.encode("utf-8"), WSMsgType.TEXT, compress=compress + ) + + async def send_bytes(self, data: bytes, compress: Optional[int] = None) -> None: + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError("data argument must be byte-ish (%r)" % type(data)) + await self._writer.send_frame(data, WSMsgType.BINARY, compress=compress) + + async def send_json( + self, + data: Any, + compress: Optional[int] = None, + *, + dumps: JSONEncoder = DEFAULT_JSON_ENCODER, + ) -> None: + await self.send_str(dumps(data), compress=compress) + + async def close(self, *, code: int = WSCloseCode.OK, message: bytes = b"") -> bool: + # we need to break `receive()` cycle first, + # `close()` may be called from different task + if self._waiting and not self._closing: + assert self._loop is not None + self._close_wait = self._loop.create_future() + self._set_closing() + self._reader.feed_data(WS_CLOSING_MESSAGE, 0) + await self._close_wait + + if self._closed: + return False + + self._set_closed() + try: + await self._writer.close(code, message) + except asyncio.CancelledError: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._response.close() + raise + except Exception as exc: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._exception = exc + self._response.close() + return True + + if self._close_code: + self._response.close() + return True + + while True: + try: + async with async_timeout.timeout(self._timeout.ws_close): + msg = await self._reader.read() + except asyncio.CancelledError: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._response.close() + raise + except Exception as exc: + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + self._exception = exc + self._response.close() + return True + + if msg.type is WSMsgType.CLOSE: + self._close_code = msg.data + self._response.close() + return True + + async def receive(self, timeout: Optional[float] = None) -> WSMessage: + receive_timeout = timeout or self._timeout.ws_receive + + while True: + if self._waiting: + raise RuntimeError("Concurrent call to receive() is not allowed") + + if self._closed: + return WS_CLOSED_MESSAGE + elif self._closing: + await self.close() + return WS_CLOSED_MESSAGE + + try: + self._waiting = True + try: + if receive_timeout: + # Entering the context manager and creating + # Timeout() object can take almost 50% of the + # run time in this loop so we avoid it if + # there is no read timeout. + async with async_timeout.timeout(receive_timeout): + msg = await self._reader.read() + else: + msg = await self._reader.read() + self._reset_heartbeat() + finally: + self._waiting = False + if self._close_wait: + set_result(self._close_wait, None) + except (asyncio.CancelledError, asyncio.TimeoutError): + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + raise + except EofStream: + self._close_code = WSCloseCode.OK + await self.close() + return WSMessage(WSMsgType.CLOSED, None, None) + except ClientError: + # Likely ServerDisconnectedError when connection is lost + self._set_closed() + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + return WS_CLOSED_MESSAGE + except WebSocketError as exc: + self._close_code = exc.code + await self.close(code=exc.code) + return WSMessage(WSMsgType.ERROR, exc, None) + except Exception as exc: + self._exception = exc + self._set_closing() + self._close_code = WSCloseCode.ABNORMAL_CLOSURE + await self.close() + return WSMessage(WSMsgType.ERROR, exc, None) + + if msg.type not in _INTERNAL_RECEIVE_TYPES: + # If its not a close/closing/ping/pong message + # we can return it immediately + return msg + + if msg.type is WSMsgType.CLOSE: + self._set_closing() + self._close_code = msg.data + if not self._closed and self._autoclose: + await self.close() + elif msg.type is WSMsgType.CLOSING: + self._set_closing() + elif msg.type is WSMsgType.PING and self._autoping: + await self.pong(msg.data) + continue + elif msg.type is WSMsgType.PONG and self._autoping: + continue + + return msg + + async def receive_str(self, *, timeout: Optional[float] = None) -> str: + msg = await self.receive(timeout) + if msg.type is not WSMsgType.TEXT: + raise WSMessageTypeError( + f"Received message {msg.type}:{msg.data!r} is not WSMsgType.TEXT" + ) + return cast(str, msg.data) + + async def receive_bytes(self, *, timeout: Optional[float] = None) -> bytes: + msg = await self.receive(timeout) + if msg.type is not WSMsgType.BINARY: + raise WSMessageTypeError( + f"Received message {msg.type}:{msg.data!r} is not WSMsgType.BINARY" + ) + return cast(bytes, msg.data) + + async def receive_json( + self, + *, + loads: JSONDecoder = DEFAULT_JSON_DECODER, + timeout: Optional[float] = None, + ) -> Any: + data = await self.receive_str(timeout=timeout) + return loads(data) + + def __aiter__(self) -> "ClientWebSocketResponse": + return self + + async def __anext__(self) -> WSMessage: + msg = await self.receive() + if msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.CLOSED): + raise StopAsyncIteration + return msg + + async def __aenter__(self) -> "ClientWebSocketResponse": + return self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + await self.close() diff --git a/lib/python3.1/site-packages/aiohttp/compression_utils.py b/lib/python3.1/site-packages/aiohttp/compression_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f08c3d9cdff0c9ee27e006b0f028a648f93a6955 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/compression_utils.py @@ -0,0 +1,278 @@ +import asyncio +import sys +import zlib +from concurrent.futures import Executor +from typing import Any, Final, Optional, Protocol, TypedDict, cast + +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + from typing import Union + + Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"] + +try: + try: + import brotlicffi as brotli + except ImportError: + import brotli + + HAS_BROTLI = True +except ImportError: # pragma: no cover + HAS_BROTLI = False + +MAX_SYNC_CHUNK_SIZE = 1024 + + +class ZLibCompressObjProtocol(Protocol): + def compress(self, data: Buffer) -> bytes: ... + def flush(self, mode: int = ..., /) -> bytes: ... + + +class ZLibDecompressObjProtocol(Protocol): + def decompress(self, data: Buffer, max_length: int = ...) -> bytes: ... + def flush(self, length: int = ..., /) -> bytes: ... + + @property + def eof(self) -> bool: ... + + +class ZLibBackendProtocol(Protocol): + MAX_WBITS: int + Z_FULL_FLUSH: int + Z_SYNC_FLUSH: int + Z_BEST_SPEED: int + Z_FINISH: int + + def compressobj( + self, + level: int = ..., + method: int = ..., + wbits: int = ..., + memLevel: int = ..., + strategy: int = ..., + zdict: Optional[Buffer] = ..., + ) -> ZLibCompressObjProtocol: ... + def decompressobj( + self, wbits: int = ..., zdict: Buffer = ... + ) -> ZLibDecompressObjProtocol: ... + + def compress( + self, data: Buffer, /, level: int = ..., wbits: int = ... + ) -> bytes: ... + def decompress( + self, data: Buffer, /, wbits: int = ..., bufsize: int = ... + ) -> bytes: ... + + +class CompressObjArgs(TypedDict, total=False): + wbits: int + strategy: int + level: int + + +class ZLibBackendWrapper: + def __init__(self, _zlib_backend: ZLibBackendProtocol): + self._zlib_backend: ZLibBackendProtocol = _zlib_backend + + @property + def name(self) -> str: + return getattr(self._zlib_backend, "__name__", "undefined") + + @property + def MAX_WBITS(self) -> int: + return self._zlib_backend.MAX_WBITS + + @property + def Z_FULL_FLUSH(self) -> int: + return self._zlib_backend.Z_FULL_FLUSH + + @property + def Z_SYNC_FLUSH(self) -> int: + return self._zlib_backend.Z_SYNC_FLUSH + + @property + def Z_BEST_SPEED(self) -> int: + return self._zlib_backend.Z_BEST_SPEED + + @property + def Z_FINISH(self) -> int: + return self._zlib_backend.Z_FINISH + + def compressobj(self, *args: Any, **kwargs: Any) -> ZLibCompressObjProtocol: + return self._zlib_backend.compressobj(*args, **kwargs) + + def decompressobj(self, *args: Any, **kwargs: Any) -> ZLibDecompressObjProtocol: + return self._zlib_backend.decompressobj(*args, **kwargs) + + def compress(self, data: Buffer, *args: Any, **kwargs: Any) -> bytes: + return self._zlib_backend.compress(data, *args, **kwargs) + + def decompress(self, data: Buffer, *args: Any, **kwargs: Any) -> bytes: + return self._zlib_backend.decompress(data, *args, **kwargs) + + # Everything not explicitly listed in the Protocol we just pass through + def __getattr__(self, attrname: str) -> Any: + return getattr(self._zlib_backend, attrname) + + +ZLibBackend: ZLibBackendWrapper = ZLibBackendWrapper(zlib) + + +def set_zlib_backend(new_zlib_backend: ZLibBackendProtocol) -> None: + ZLibBackend._zlib_backend = new_zlib_backend + + +def encoding_to_mode( + encoding: Optional[str] = None, + suppress_deflate_header: bool = False, +) -> int: + if encoding == "gzip": + return 16 + ZLibBackend.MAX_WBITS + + return -ZLibBackend.MAX_WBITS if suppress_deflate_header else ZLibBackend.MAX_WBITS + + +class ZlibBaseHandler: + def __init__( + self, + mode: int, + executor: Optional[Executor] = None, + max_sync_chunk_size: Optional[int] = MAX_SYNC_CHUNK_SIZE, + ): + self._mode = mode + self._executor = executor + self._max_sync_chunk_size = max_sync_chunk_size + + +class ZLibCompressor(ZlibBaseHandler): + def __init__( + self, + encoding: Optional[str] = None, + suppress_deflate_header: bool = False, + level: Optional[int] = None, + wbits: Optional[int] = None, + strategy: Optional[int] = None, + executor: Optional[Executor] = None, + max_sync_chunk_size: Optional[int] = MAX_SYNC_CHUNK_SIZE, + ): + super().__init__( + mode=( + encoding_to_mode(encoding, suppress_deflate_header) + if wbits is None + else wbits + ), + executor=executor, + max_sync_chunk_size=max_sync_chunk_size, + ) + self._zlib_backend: Final = ZLibBackendWrapper(ZLibBackend._zlib_backend) + + kwargs: CompressObjArgs = {} + kwargs["wbits"] = self._mode + if strategy is not None: + kwargs["strategy"] = strategy + if level is not None: + kwargs["level"] = level + self._compressor = self._zlib_backend.compressobj(**kwargs) + self._compress_lock = asyncio.Lock() + + def compress_sync(self, data: bytes) -> bytes: + return self._compressor.compress(data) + + async def compress(self, data: bytes) -> bytes: + """Compress the data and returned the compressed bytes. + + Note that flush() must be called after the last call to compress() + + If the data size is large than the max_sync_chunk_size, the compression + will be done in the executor. Otherwise, the compression will be done + in the event loop. + """ + async with self._compress_lock: + # To ensure the stream is consistent in the event + # there are multiple writers, we need to lock + # the compressor so that only one writer can + # compress at a time. + if ( + self._max_sync_chunk_size is not None + and len(data) > self._max_sync_chunk_size + ): + return await asyncio.get_running_loop().run_in_executor( + self._executor, self._compressor.compress, data + ) + return self.compress_sync(data) + + def flush(self, mode: Optional[int] = None) -> bytes: + return self._compressor.flush( + mode if mode is not None else self._zlib_backend.Z_FINISH + ) + + +class ZLibDecompressor(ZlibBaseHandler): + def __init__( + self, + encoding: Optional[str] = None, + suppress_deflate_header: bool = False, + executor: Optional[Executor] = None, + max_sync_chunk_size: Optional[int] = MAX_SYNC_CHUNK_SIZE, + ): + super().__init__( + mode=encoding_to_mode(encoding, suppress_deflate_header), + executor=executor, + max_sync_chunk_size=max_sync_chunk_size, + ) + self._zlib_backend: Final = ZLibBackendWrapper(ZLibBackend._zlib_backend) + self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode) + + def decompress_sync(self, data: bytes, max_length: int = 0) -> bytes: + return self._decompressor.decompress(data, max_length) + + async def decompress(self, data: bytes, max_length: int = 0) -> bytes: + """Decompress the data and return the decompressed bytes. + + If the data size is large than the max_sync_chunk_size, the decompression + will be done in the executor. Otherwise, the decompression will be done + in the event loop. + """ + if ( + self._max_sync_chunk_size is not None + and len(data) > self._max_sync_chunk_size + ): + return await asyncio.get_running_loop().run_in_executor( + self._executor, self._decompressor.decompress, data, max_length + ) + return self.decompress_sync(data, max_length) + + def flush(self, length: int = 0) -> bytes: + return ( + self._decompressor.flush(length) + if length > 0 + else self._decompressor.flush() + ) + + @property + def eof(self) -> bool: + return self._decompressor.eof + + +class BrotliDecompressor: + # Supports both 'brotlipy' and 'Brotli' packages + # since they share an import name. The top branches + # are for 'brotlipy' and bottom branches for 'Brotli' + def __init__(self) -> None: + if not HAS_BROTLI: + raise RuntimeError( + "The brotli decompression is not available. " + "Please install `Brotli` module" + ) + self._obj = brotli.Decompressor() + + def decompress_sync(self, data: bytes) -> bytes: + if hasattr(self._obj, "decompress"): + return cast(bytes, self._obj.decompress(data)) + return cast(bytes, self._obj.process(data)) + + def flush(self) -> bytes: + if hasattr(self._obj, "flush"): + return cast(bytes, self._obj.flush()) + return b"" diff --git a/lib/python3.1/site-packages/aiohttp/connector.py b/lib/python3.1/site-packages/aiohttp/connector.py new file mode 100644 index 0000000000000000000000000000000000000000..0fbacde3b42f232d2ddc2b6c3967a184ba22c252 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/connector.py @@ -0,0 +1,1834 @@ +import asyncio +import functools +import random +import socket +import sys +import traceback +import warnings +from collections import OrderedDict, defaultdict, deque +from contextlib import suppress +from http import HTTPStatus +from itertools import chain, cycle, islice +from time import monotonic +from types import TracebackType +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + DefaultDict, + Deque, + Dict, + Iterator, + List, + Literal, + Optional, + Sequence, + Set, + Tuple, + Type, + Union, + cast, +) + +import aiohappyeyeballs +from aiohappyeyeballs import AddrInfoType, SocketFactoryType + +from . import hdrs, helpers +from .abc import AbstractResolver, ResolveResult +from .client_exceptions import ( + ClientConnectionError, + ClientConnectorCertificateError, + ClientConnectorDNSError, + ClientConnectorError, + ClientConnectorSSLError, + ClientHttpProxyError, + ClientProxyConnectionError, + ServerFingerprintMismatch, + UnixClientConnectorError, + cert_errors, + ssl_errors, +) +from .client_proto import ResponseHandler +from .client_reqrep import ClientRequest, Fingerprint, _merge_ssl_params +from .helpers import ( + _SENTINEL, + ceil_timeout, + is_ip_address, + noop, + sentinel, + set_exception, + set_result, +) +from .log import client_logger +from .resolver import DefaultResolver + +if sys.version_info >= (3, 12): + from collections.abc import Buffer +else: + Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"] + +if TYPE_CHECKING: + import ssl + + SSLContext = ssl.SSLContext +else: + try: + import ssl + + SSLContext = ssl.SSLContext + except ImportError: # pragma: no cover + ssl = None # type: ignore[assignment] + SSLContext = object # type: ignore[misc,assignment] + +EMPTY_SCHEMA_SET = frozenset({""}) +HTTP_SCHEMA_SET = frozenset({"http", "https"}) +WS_SCHEMA_SET = frozenset({"ws", "wss"}) + +HTTP_AND_EMPTY_SCHEMA_SET = HTTP_SCHEMA_SET | EMPTY_SCHEMA_SET +HIGH_LEVEL_SCHEMA_SET = HTTP_AND_EMPTY_SCHEMA_SET | WS_SCHEMA_SET + +NEEDS_CLEANUP_CLOSED = (3, 13, 0) <= sys.version_info < ( + 3, + 13, + 1, +) or sys.version_info < (3, 12, 7) +# Cleanup closed is no longer needed after https://github.com/python/cpython/pull/118960 +# which first appeared in Python 3.12.7 and 3.13.1 + + +__all__ = ( + "BaseConnector", + "TCPConnector", + "UnixConnector", + "NamedPipeConnector", + "AddrInfoType", + "SocketFactoryType", +) + + +if TYPE_CHECKING: + from .client import ClientTimeout + from .client_reqrep import ConnectionKey + from .tracing import Trace + + +class _DeprecationWaiter: + __slots__ = ("_awaitable", "_awaited") + + def __init__(self, awaitable: Awaitable[Any]) -> None: + self._awaitable = awaitable + self._awaited = False + + def __await__(self) -> Any: + self._awaited = True + return self._awaitable.__await__() + + def __del__(self) -> None: + if not self._awaited: + warnings.warn( + "Connector.close() is a coroutine, " + "please use await connector.close()", + DeprecationWarning, + ) + + +async def _wait_for_close(waiters: List[Awaitable[object]]) -> None: + """Wait for all waiters to finish closing.""" + results = await asyncio.gather(*waiters, return_exceptions=True) + for res in results: + if isinstance(res, Exception): + client_logger.debug("Error while closing connector: %r", res) + + +class Connection: + + _source_traceback = None + + def __init__( + self, + connector: "BaseConnector", + key: "ConnectionKey", + protocol: ResponseHandler, + loop: asyncio.AbstractEventLoop, + ) -> None: + self._key = key + self._connector = connector + self._loop = loop + self._protocol: Optional[ResponseHandler] = protocol + self._callbacks: List[Callable[[], None]] = [] + + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + def __repr__(self) -> str: + return f"Connection<{self._key}>" + + def __del__(self, _warnings: Any = warnings) -> None: + if self._protocol is not None: + kwargs = {"source": self} + _warnings.warn(f"Unclosed connection {self!r}", ResourceWarning, **kwargs) + if self._loop.is_closed(): + return + + self._connector._release(self._key, self._protocol, should_close=True) + + context = {"client_connection": self, "message": "Unclosed connection"} + if self._source_traceback is not None: + context["source_traceback"] = self._source_traceback + self._loop.call_exception_handler(context) + + def __bool__(self) -> Literal[True]: + """Force subclasses to not be falsy, to make checks simpler.""" + return True + + @property + def loop(self) -> asyncio.AbstractEventLoop: + warnings.warn( + "connector.loop property is deprecated", DeprecationWarning, stacklevel=2 + ) + return self._loop + + @property + def transport(self) -> Optional[asyncio.Transport]: + if self._protocol is None: + return None + return self._protocol.transport + + @property + def protocol(self) -> Optional[ResponseHandler]: + return self._protocol + + def add_callback(self, callback: Callable[[], None]) -> None: + if callback is not None: + self._callbacks.append(callback) + + def _notify_release(self) -> None: + callbacks, self._callbacks = self._callbacks[:], [] + + for cb in callbacks: + with suppress(Exception): + cb() + + def close(self) -> None: + self._notify_release() + + if self._protocol is not None: + self._connector._release(self._key, self._protocol, should_close=True) + self._protocol = None + + def release(self) -> None: + self._notify_release() + + if self._protocol is not None: + self._connector._release(self._key, self._protocol) + self._protocol = None + + @property + def closed(self) -> bool: + return self._protocol is None or not self._protocol.is_connected() + + +class _ConnectTunnelConnection(Connection): + """Special connection wrapper for CONNECT tunnels that must never be pooled. + + This connection wraps the proxy connection that will be upgraded with TLS. + It must never be released to the pool because: + 1. Its 'closed' future will never complete, causing session.close() to hang + 2. It represents an intermediate state, not a reusable connection + 3. The real connection (with TLS) will be created separately + """ + + def release(self) -> None: + """Do nothing - don't pool or close the connection. + + These connections are an intermediate state during the CONNECT tunnel + setup and will be cleaned up naturally after the TLS upgrade. If they + were to be pooled, they would never be properly closed, causing + session.close() to wait forever for their 'closed' future. + """ + + +class _TransportPlaceholder: + """placeholder for BaseConnector.connect function""" + + __slots__ = ("closed", "transport") + + def __init__(self, closed_future: asyncio.Future[Optional[Exception]]) -> None: + """Initialize a placeholder for a transport.""" + self.closed = closed_future + self.transport = None + + def close(self) -> None: + """Close the placeholder.""" + + def abort(self) -> None: + """Abort the placeholder (does nothing).""" + + +class BaseConnector: + """Base connector class. + + keepalive_timeout - (optional) Keep-alive timeout. + force_close - Set to True to force close and do reconnect + after each request (and between redirects). + limit - The total number of simultaneous connections. + limit_per_host - Number of simultaneous connections to one host. + enable_cleanup_closed - Enables clean-up closed ssl transports. + Disabled by default. + timeout_ceil_threshold - Trigger ceiling of timeout values when + it's above timeout_ceil_threshold. + loop - Optional event loop. + """ + + _closed = True # prevent AttributeError in __del__ if ctor was failed + _source_traceback = None + + # abort transport after 2 seconds (cleanup broken connections) + _cleanup_closed_period = 2.0 + + allowed_protocol_schema_set = HIGH_LEVEL_SCHEMA_SET + + def __init__( + self, + *, + keepalive_timeout: Union[object, None, float] = sentinel, + force_close: bool = False, + limit: int = 100, + limit_per_host: int = 0, + enable_cleanup_closed: bool = False, + loop: Optional[asyncio.AbstractEventLoop] = None, + timeout_ceil_threshold: float = 5, + ) -> None: + + if force_close: + if keepalive_timeout is not None and keepalive_timeout is not sentinel: + raise ValueError( + "keepalive_timeout cannot be set if force_close is True" + ) + else: + if keepalive_timeout is sentinel: + keepalive_timeout = 15.0 + + loop = loop or asyncio.get_running_loop() + self._timeout_ceil_threshold = timeout_ceil_threshold + + self._closed = False + if loop.get_debug(): + self._source_traceback = traceback.extract_stack(sys._getframe(1)) + + # Connection pool of reusable connections. + # We use a deque to store connections because it has O(1) popleft() + # and O(1) append() operations to implement a FIFO queue. + self._conns: DefaultDict[ + ConnectionKey, Deque[Tuple[ResponseHandler, float]] + ] = defaultdict(deque) + self._limit = limit + self._limit_per_host = limit_per_host + self._acquired: Set[ResponseHandler] = set() + self._acquired_per_host: DefaultDict[ConnectionKey, Set[ResponseHandler]] = ( + defaultdict(set) + ) + self._keepalive_timeout = cast(float, keepalive_timeout) + self._force_close = force_close + + # {host_key: FIFO list of waiters} + # The FIFO is implemented with an OrderedDict with None keys because + # python does not have an ordered set. + self._waiters: DefaultDict[ + ConnectionKey, OrderedDict[asyncio.Future[None], None] + ] = defaultdict(OrderedDict) + + self._loop = loop + self._factory = functools.partial(ResponseHandler, loop=loop) + + # start keep-alive connection cleanup task + self._cleanup_handle: Optional[asyncio.TimerHandle] = None + + # start cleanup closed transports task + self._cleanup_closed_handle: Optional[asyncio.TimerHandle] = None + + if enable_cleanup_closed and not NEEDS_CLEANUP_CLOSED: + warnings.warn( + "enable_cleanup_closed ignored because " + "https://github.com/python/cpython/pull/118960 is fixed " + f"in Python version {sys.version_info}", + DeprecationWarning, + stacklevel=2, + ) + enable_cleanup_closed = False + + self._cleanup_closed_disabled = not enable_cleanup_closed + self._cleanup_closed_transports: List[Optional[asyncio.Transport]] = [] + self._placeholder_future: asyncio.Future[Optional[Exception]] = ( + loop.create_future() + ) + self._placeholder_future.set_result(None) + self._cleanup_closed() + + def __del__(self, _warnings: Any = warnings) -> None: + if self._closed: + return + if not self._conns: + return + + conns = [repr(c) for c in self._conns.values()] + + self._close() + + kwargs = {"source": self} + _warnings.warn(f"Unclosed connector {self!r}", ResourceWarning, **kwargs) + context = { + "connector": self, + "connections": conns, + "message": "Unclosed connector", + } + if self._source_traceback is not None: + context["source_traceback"] = self._source_traceback + self._loop.call_exception_handler(context) + + def __enter__(self) -> "BaseConnector": + warnings.warn( + '"with Connector():" is deprecated, ' + 'use "async with Connector():" instead', + DeprecationWarning, + ) + return self + + def __exit__(self, *exc: Any) -> None: + self._close() + + async def __aenter__(self) -> "BaseConnector": + return self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]] = None, + exc_value: Optional[BaseException] = None, + exc_traceback: Optional[TracebackType] = None, + ) -> None: + await self.close() + + @property + def force_close(self) -> bool: + """Ultimately close connection on releasing if True.""" + return self._force_close + + @property + def limit(self) -> int: + """The total number for simultaneous connections. + + If limit is 0 the connector has no limit. + The default limit size is 100. + """ + return self._limit + + @property + def limit_per_host(self) -> int: + """The limit for simultaneous connections to the same endpoint. + + Endpoints are the same if they are have equal + (host, port, is_ssl) triple. + """ + return self._limit_per_host + + def _cleanup(self) -> None: + """Cleanup unused transports.""" + if self._cleanup_handle: + self._cleanup_handle.cancel() + # _cleanup_handle should be unset, otherwise _release() will not + # recreate it ever! + self._cleanup_handle = None + + now = monotonic() + timeout = self._keepalive_timeout + + if self._conns: + connections = defaultdict(deque) + deadline = now - timeout + for key, conns in self._conns.items(): + alive: Deque[Tuple[ResponseHandler, float]] = deque() + for proto, use_time in conns: + if proto.is_connected() and use_time - deadline >= 0: + alive.append((proto, use_time)) + continue + transport = proto.transport + proto.close() + if not self._cleanup_closed_disabled and key.is_ssl: + self._cleanup_closed_transports.append(transport) + + if alive: + connections[key] = alive + + self._conns = connections + + if self._conns: + self._cleanup_handle = helpers.weakref_handle( + self, + "_cleanup", + timeout, + self._loop, + timeout_ceil_threshold=self._timeout_ceil_threshold, + ) + + def _cleanup_closed(self) -> None: + """Double confirmation for transport close. + + Some broken ssl servers may leave socket open without proper close. + """ + if self._cleanup_closed_handle: + self._cleanup_closed_handle.cancel() + + for transport in self._cleanup_closed_transports: + if transport is not None: + transport.abort() + + self._cleanup_closed_transports = [] + + if not self._cleanup_closed_disabled: + self._cleanup_closed_handle = helpers.weakref_handle( + self, + "_cleanup_closed", + self._cleanup_closed_period, + self._loop, + timeout_ceil_threshold=self._timeout_ceil_threshold, + ) + + def close(self, *, abort_ssl: bool = False) -> Awaitable[None]: + """Close all opened transports. + + :param abort_ssl: If True, SSL connections will be aborted immediately + without performing the shutdown handshake. This provides + faster cleanup at the cost of less graceful disconnection. + """ + if not (waiters := self._close(abort_ssl=abort_ssl)): + # If there are no connections to close, we can return a noop + # awaitable to avoid scheduling a task on the event loop. + return _DeprecationWaiter(noop()) + coro = _wait_for_close(waiters) + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to close connections + # immediately to avoid having to schedule the task on the event loop. + task = asyncio.Task(coro, loop=self._loop, eager_start=True) + else: + task = self._loop.create_task(coro) + return _DeprecationWaiter(task) + + def _close(self, *, abort_ssl: bool = False) -> List[Awaitable[object]]: + waiters: List[Awaitable[object]] = [] + + if self._closed: + return waiters + + self._closed = True + + try: + if self._loop.is_closed(): + return waiters + + # cancel cleanup task + if self._cleanup_handle: + self._cleanup_handle.cancel() + + # cancel cleanup close task + if self._cleanup_closed_handle: + self._cleanup_closed_handle.cancel() + + for data in self._conns.values(): + for proto, _ in data: + if ( + abort_ssl + and proto.transport + and proto.transport.get_extra_info("sslcontext") is not None + ): + proto.abort() + else: + proto.close() + if closed := proto.closed: + waiters.append(closed) + + for proto in self._acquired: + if ( + abort_ssl + and proto.transport + and proto.transport.get_extra_info("sslcontext") is not None + ): + proto.abort() + else: + proto.close() + if closed := proto.closed: + waiters.append(closed) + + for transport in self._cleanup_closed_transports: + if transport is not None: + transport.abort() + + return waiters + + finally: + self._conns.clear() + self._acquired.clear() + for keyed_waiters in self._waiters.values(): + for keyed_waiter in keyed_waiters: + keyed_waiter.cancel() + self._waiters.clear() + self._cleanup_handle = None + self._cleanup_closed_transports.clear() + self._cleanup_closed_handle = None + + @property + def closed(self) -> bool: + """Is connector closed. + + A readonly property. + """ + return self._closed + + def _available_connections(self, key: "ConnectionKey") -> int: + """ + Return number of available connections. + + The limit, limit_per_host and the connection key are taken into account. + + If it returns less than 1 means that there are no connections + available. + """ + # check total available connections + # If there are no limits, this will always return 1 + total_remain = 1 + + if self._limit and (total_remain := self._limit - len(self._acquired)) <= 0: + return total_remain + + # check limit per host + if host_remain := self._limit_per_host: + if acquired := self._acquired_per_host.get(key): + host_remain -= len(acquired) + if total_remain > host_remain: + return host_remain + + return total_remain + + async def connect( + self, req: ClientRequest, traces: List["Trace"], timeout: "ClientTimeout" + ) -> Connection: + """Get from pool or create new connection.""" + key = req.connection_key + if (conn := await self._get(key, traces)) is not None: + # If we do not have to wait and we can get a connection from the pool + # we can avoid the timeout ceil logic and directly return the connection + return conn + + async with ceil_timeout(timeout.connect, timeout.ceil_threshold): + if self._available_connections(key) <= 0: + await self._wait_for_available_connection(key, traces) + if (conn := await self._get(key, traces)) is not None: + return conn + + placeholder = cast( + ResponseHandler, _TransportPlaceholder(self._placeholder_future) + ) + self._acquired.add(placeholder) + if self._limit_per_host: + self._acquired_per_host[key].add(placeholder) + + try: + # Traces are done inside the try block to ensure that the + # that the placeholder is still cleaned up if an exception + # is raised. + if traces: + for trace in traces: + await trace.send_connection_create_start() + proto = await self._create_connection(req, traces, timeout) + if traces: + for trace in traces: + await trace.send_connection_create_end() + except BaseException: + self._release_acquired(key, placeholder) + raise + else: + if self._closed: + proto.close() + raise ClientConnectionError("Connector is closed.") + + # The connection was successfully created, drop the placeholder + # and add the real connection to the acquired set. There should + # be no awaits after the proto is added to the acquired set + # to ensure that the connection is not left in the acquired set + # on cancellation. + self._acquired.remove(placeholder) + self._acquired.add(proto) + if self._limit_per_host: + acquired_per_host = self._acquired_per_host[key] + acquired_per_host.remove(placeholder) + acquired_per_host.add(proto) + return Connection(self, key, proto, self._loop) + + async def _wait_for_available_connection( + self, key: "ConnectionKey", traces: List["Trace"] + ) -> None: + """Wait for an available connection slot.""" + # We loop here because there is a race between + # the connection limit check and the connection + # being acquired. If the connection is acquired + # between the check and the await statement, we + # need to loop again to check if the connection + # slot is still available. + attempts = 0 + while True: + fut: asyncio.Future[None] = self._loop.create_future() + keyed_waiters = self._waiters[key] + keyed_waiters[fut] = None + if attempts: + # If we have waited before, we need to move the waiter + # to the front of the queue as otherwise we might get + # starved and hit the timeout. + keyed_waiters.move_to_end(fut, last=False) + + try: + # Traces happen in the try block to ensure that the + # the waiter is still cleaned up if an exception is raised. + if traces: + for trace in traces: + await trace.send_connection_queued_start() + await fut + if traces: + for trace in traces: + await trace.send_connection_queued_end() + finally: + # pop the waiter from the queue if its still + # there and not already removed by _release_waiter + keyed_waiters.pop(fut, None) + if not self._waiters.get(key, True): + del self._waiters[key] + + if self._available_connections(key) > 0: + break + attempts += 1 + + async def _get( + self, key: "ConnectionKey", traces: List["Trace"] + ) -> Optional[Connection]: + """Get next reusable connection for the key or None. + + The connection will be marked as acquired. + """ + if (conns := self._conns.get(key)) is None: + return None + + t1 = monotonic() + while conns: + proto, t0 = conns.popleft() + # We will we reuse the connection if its connected and + # the keepalive timeout has not been exceeded + if proto.is_connected() and t1 - t0 <= self._keepalive_timeout: + if not conns: + # The very last connection was reclaimed: drop the key + del self._conns[key] + self._acquired.add(proto) + if self._limit_per_host: + self._acquired_per_host[key].add(proto) + if traces: + for trace in traces: + try: + await trace.send_connection_reuseconn() + except BaseException: + self._release_acquired(key, proto) + raise + return Connection(self, key, proto, self._loop) + + # Connection cannot be reused, close it + transport = proto.transport + proto.close() + # only for SSL transports + if not self._cleanup_closed_disabled and key.is_ssl: + self._cleanup_closed_transports.append(transport) + + # No more connections: drop the key + del self._conns[key] + return None + + def _release_waiter(self) -> None: + """ + Iterates over all waiters until one to be released is found. + + The one to be released is not finished and + belongs to a host that has available connections. + """ + if not self._waiters: + return + + # Having the dict keys ordered this avoids to iterate + # at the same order at each call. + queues = list(self._waiters) + random.shuffle(queues) + + for key in queues: + if self._available_connections(key) < 1: + continue + + waiters = self._waiters[key] + while waiters: + waiter, _ = waiters.popitem(last=False) + if not waiter.done(): + waiter.set_result(None) + return + + def _release_acquired(self, key: "ConnectionKey", proto: ResponseHandler) -> None: + """Release acquired connection.""" + if self._closed: + # acquired connection is already released on connector closing + return + + self._acquired.discard(proto) + if self._limit_per_host and (conns := self._acquired_per_host.get(key)): + conns.discard(proto) + if not conns: + del self._acquired_per_host[key] + self._release_waiter() + + def _release( + self, + key: "ConnectionKey", + protocol: ResponseHandler, + *, + should_close: bool = False, + ) -> None: + if self._closed: + # acquired connection is already released on connector closing + return + + self._release_acquired(key, protocol) + + if self._force_close or should_close or protocol.should_close: + transport = protocol.transport + protocol.close() + + if key.is_ssl and not self._cleanup_closed_disabled: + self._cleanup_closed_transports.append(transport) + return + + self._conns[key].append((protocol, monotonic())) + + if self._cleanup_handle is None: + self._cleanup_handle = helpers.weakref_handle( + self, + "_cleanup", + self._keepalive_timeout, + self._loop, + timeout_ceil_threshold=self._timeout_ceil_threshold, + ) + + async def _create_connection( + self, req: ClientRequest, traces: List["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + raise NotImplementedError() + + +class _DNSCacheTable: + def __init__(self, ttl: Optional[float] = None) -> None: + self._addrs_rr: Dict[Tuple[str, int], Tuple[Iterator[ResolveResult], int]] = {} + self._timestamps: Dict[Tuple[str, int], float] = {} + self._ttl = ttl + + def __contains__(self, host: object) -> bool: + return host in self._addrs_rr + + def add(self, key: Tuple[str, int], addrs: List[ResolveResult]) -> None: + self._addrs_rr[key] = (cycle(addrs), len(addrs)) + + if self._ttl is not None: + self._timestamps[key] = monotonic() + + def remove(self, key: Tuple[str, int]) -> None: + self._addrs_rr.pop(key, None) + + if self._ttl is not None: + self._timestamps.pop(key, None) + + def clear(self) -> None: + self._addrs_rr.clear() + self._timestamps.clear() + + def next_addrs(self, key: Tuple[str, int]) -> List[ResolveResult]: + loop, length = self._addrs_rr[key] + addrs = list(islice(loop, length)) + # Consume one more element to shift internal state of `cycle` + next(loop) + return addrs + + def expired(self, key: Tuple[str, int]) -> bool: + if self._ttl is None: + return False + + return self._timestamps[key] + self._ttl < monotonic() + + +def _make_ssl_context(verified: bool) -> SSLContext: + """Create SSL context. + + This method is not async-friendly and should be called from a thread + because it will load certificates from disk and do other blocking I/O. + """ + if ssl is None: + # No ssl support + return None + if verified: + sslcontext = ssl.create_default_context() + else: + sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslcontext.options |= ssl.OP_NO_SSLv2 + sslcontext.options |= ssl.OP_NO_SSLv3 + sslcontext.check_hostname = False + sslcontext.verify_mode = ssl.CERT_NONE + sslcontext.options |= ssl.OP_NO_COMPRESSION + sslcontext.set_default_verify_paths() + sslcontext.set_alpn_protocols(("http/1.1",)) + return sslcontext + + +# The default SSLContext objects are created at import time +# since they do blocking I/O to load certificates from disk, +# and imports should always be done before the event loop starts +# or in a thread. +_SSL_CONTEXT_VERIFIED = _make_ssl_context(True) +_SSL_CONTEXT_UNVERIFIED = _make_ssl_context(False) + + +class TCPConnector(BaseConnector): + """TCP connector. + + verify_ssl - Set to True to check ssl certifications. + fingerprint - Pass the binary sha256 + digest of the expected certificate in DER format to verify + that the certificate the server presents matches. See also + https://en.wikipedia.org/wiki/HTTP_Public_Key_Pinning + resolver - Enable DNS lookups and use this + resolver + use_dns_cache - Use memory cache for DNS lookups. + ttl_dns_cache - Max seconds having cached a DNS entry, None forever. + family - socket address family + local_addr - local tuple of (host, port) to bind socket to + + keepalive_timeout - (optional) Keep-alive timeout. + force_close - Set to True to force close and do reconnect + after each request (and between redirects). + limit - The total number of simultaneous connections. + limit_per_host - Number of simultaneous connections to one host. + enable_cleanup_closed - Enables clean-up closed ssl transports. + Disabled by default. + happy_eyeballs_delay - This is the “Connection Attempt Delay” + as defined in RFC 8305. To disable + the happy eyeballs algorithm, set to None. + interleave - “First Address Family Count” as defined in RFC 8305 + loop - Optional event loop. + socket_factory - A SocketFactoryType function that, if supplied, + will be used to create sockets given an + AddrInfoType. + ssl_shutdown_timeout - DEPRECATED. Will be removed in aiohttp 4.0. + Grace period for SSL shutdown handshake on TLS + connections. Default is 0 seconds (immediate abort). + This parameter allowed for a clean SSL shutdown by + notifying the remote peer of connection closure, + while avoiding excessive delays during connector cleanup. + Note: Only takes effect on Python 3.11+. + """ + + allowed_protocol_schema_set = HIGH_LEVEL_SCHEMA_SET | frozenset({"tcp"}) + + def __init__( + self, + *, + verify_ssl: bool = True, + fingerprint: Optional[bytes] = None, + use_dns_cache: bool = True, + ttl_dns_cache: Optional[int] = 10, + family: socket.AddressFamily = socket.AddressFamily.AF_UNSPEC, + ssl_context: Optional[SSLContext] = None, + ssl: Union[bool, Fingerprint, SSLContext] = True, + local_addr: Optional[Tuple[str, int]] = None, + resolver: Optional[AbstractResolver] = None, + keepalive_timeout: Union[None, float, object] = sentinel, + force_close: bool = False, + limit: int = 100, + limit_per_host: int = 0, + enable_cleanup_closed: bool = False, + loop: Optional[asyncio.AbstractEventLoop] = None, + timeout_ceil_threshold: float = 5, + happy_eyeballs_delay: Optional[float] = 0.25, + interleave: Optional[int] = None, + socket_factory: Optional[SocketFactoryType] = None, + ssl_shutdown_timeout: Union[_SENTINEL, None, float] = sentinel, + ): + super().__init__( + keepalive_timeout=keepalive_timeout, + force_close=force_close, + limit=limit, + limit_per_host=limit_per_host, + enable_cleanup_closed=enable_cleanup_closed, + loop=loop, + timeout_ceil_threshold=timeout_ceil_threshold, + ) + + self._ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint) + + self._resolver: AbstractResolver + if resolver is None: + self._resolver = DefaultResolver(loop=self._loop) + self._resolver_owner = True + else: + self._resolver = resolver + self._resolver_owner = False + + self._use_dns_cache = use_dns_cache + self._cached_hosts = _DNSCacheTable(ttl=ttl_dns_cache) + self._throttle_dns_futures: Dict[ + Tuple[str, int], Set["asyncio.Future[None]"] + ] = {} + self._family = family + self._local_addr_infos = aiohappyeyeballs.addr_to_addr_infos(local_addr) + self._happy_eyeballs_delay = happy_eyeballs_delay + self._interleave = interleave + self._resolve_host_tasks: Set["asyncio.Task[List[ResolveResult]]"] = set() + self._socket_factory = socket_factory + self._ssl_shutdown_timeout: Optional[float] + # Handle ssl_shutdown_timeout with warning for Python < 3.11 + if ssl_shutdown_timeout is sentinel: + self._ssl_shutdown_timeout = 0 + else: + # Deprecation warning for ssl_shutdown_timeout parameter + warnings.warn( + "The ssl_shutdown_timeout parameter is deprecated and will be removed in aiohttp 4.0", + DeprecationWarning, + stacklevel=2, + ) + if ( + sys.version_info < (3, 11) + and ssl_shutdown_timeout is not None + and ssl_shutdown_timeout != 0 + ): + warnings.warn( + f"ssl_shutdown_timeout={ssl_shutdown_timeout} is ignored on Python < 3.11; " + "only ssl_shutdown_timeout=0 is supported. The timeout will be ignored.", + RuntimeWarning, + stacklevel=2, + ) + self._ssl_shutdown_timeout = ssl_shutdown_timeout + + def _close(self, *, abort_ssl: bool = False) -> List[Awaitable[object]]: + """Close all ongoing DNS calls.""" + for fut in chain.from_iterable(self._throttle_dns_futures.values()): + fut.cancel() + + waiters = super()._close(abort_ssl=abort_ssl) + + for t in self._resolve_host_tasks: + t.cancel() + waiters.append(t) + + return waiters + + async def close(self, *, abort_ssl: bool = False) -> None: + """ + Close all opened transports. + + :param abort_ssl: If True, SSL connections will be aborted immediately + without performing the shutdown handshake. If False (default), + the behavior is determined by ssl_shutdown_timeout: + - If ssl_shutdown_timeout=0: connections are aborted + - If ssl_shutdown_timeout>0: graceful shutdown is performed + """ + if self._resolver_owner: + await self._resolver.close() + # Use abort_ssl param if explicitly set, otherwise use ssl_shutdown_timeout default + await super().close(abort_ssl=abort_ssl or self._ssl_shutdown_timeout == 0) + + @property + def family(self) -> int: + """Socket family like AF_INET.""" + return self._family + + @property + def use_dns_cache(self) -> bool: + """True if local DNS caching is enabled.""" + return self._use_dns_cache + + def clear_dns_cache( + self, host: Optional[str] = None, port: Optional[int] = None + ) -> None: + """Remove specified host/port or clear all dns local cache.""" + if host is not None and port is not None: + self._cached_hosts.remove((host, port)) + elif host is not None or port is not None: + raise ValueError("either both host and port or none of them are allowed") + else: + self._cached_hosts.clear() + + async def _resolve_host( + self, host: str, port: int, traces: Optional[Sequence["Trace"]] = None + ) -> List[ResolveResult]: + """Resolve host and return list of addresses.""" + if is_ip_address(host): + return [ + { + "hostname": host, + "host": host, + "port": port, + "family": self._family, + "proto": 0, + "flags": 0, + } + ] + + if not self._use_dns_cache: + + if traces: + for trace in traces: + await trace.send_dns_resolvehost_start(host) + + res = await self._resolver.resolve(host, port, family=self._family) + + if traces: + for trace in traces: + await trace.send_dns_resolvehost_end(host) + + return res + + key = (host, port) + if key in self._cached_hosts and not self._cached_hosts.expired(key): + # get result early, before any await (#4014) + result = self._cached_hosts.next_addrs(key) + + if traces: + for trace in traces: + await trace.send_dns_cache_hit(host) + return result + + futures: Set["asyncio.Future[None]"] + # + # If multiple connectors are resolving the same host, we wait + # for the first one to resolve and then use the result for all of them. + # We use a throttle to ensure that we only resolve the host once + # and then use the result for all the waiters. + # + if key in self._throttle_dns_futures: + # get futures early, before any await (#4014) + futures = self._throttle_dns_futures[key] + future: asyncio.Future[None] = self._loop.create_future() + futures.add(future) + if traces: + for trace in traces: + await trace.send_dns_cache_hit(host) + try: + await future + finally: + futures.discard(future) + return self._cached_hosts.next_addrs(key) + + # update dict early, before any await (#4014) + self._throttle_dns_futures[key] = futures = set() + # In this case we need to create a task to ensure that we can shield + # the task from cancellation as cancelling this lookup should not cancel + # the underlying lookup or else the cancel event will get broadcast to + # all the waiters across all connections. + # + coro = self._resolve_host_with_throttle(key, host, port, futures, traces) + loop = asyncio.get_running_loop() + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to send immediately + resolved_host_task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + resolved_host_task = loop.create_task(coro) + + if not resolved_host_task.done(): + self._resolve_host_tasks.add(resolved_host_task) + resolved_host_task.add_done_callback(self._resolve_host_tasks.discard) + + try: + return await asyncio.shield(resolved_host_task) + except asyncio.CancelledError: + + def drop_exception(fut: "asyncio.Future[List[ResolveResult]]") -> None: + with suppress(Exception, asyncio.CancelledError): + fut.result() + + resolved_host_task.add_done_callback(drop_exception) + raise + + async def _resolve_host_with_throttle( + self, + key: Tuple[str, int], + host: str, + port: int, + futures: Set["asyncio.Future[None]"], + traces: Optional[Sequence["Trace"]], + ) -> List[ResolveResult]: + """Resolve host and set result for all waiters. + + This method must be run in a task and shielded from cancellation + to avoid cancelling the underlying lookup. + """ + try: + if traces: + for trace in traces: + await trace.send_dns_cache_miss(host) + + for trace in traces: + await trace.send_dns_resolvehost_start(host) + + addrs = await self._resolver.resolve(host, port, family=self._family) + if traces: + for trace in traces: + await trace.send_dns_resolvehost_end(host) + + self._cached_hosts.add(key, addrs) + for fut in futures: + set_result(fut, None) + except BaseException as e: + # any DNS exception is set for the waiters to raise the same exception. + # This coro is always run in task that is shielded from cancellation so + # we should never be propagating cancellation here. + for fut in futures: + set_exception(fut, e) + raise + finally: + self._throttle_dns_futures.pop(key) + + return self._cached_hosts.next_addrs(key) + + async def _create_connection( + self, req: ClientRequest, traces: List["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + """Create connection. + + Has same keyword arguments as BaseEventLoop.create_connection. + """ + if req.proxy: + _, proto = await self._create_proxy_connection(req, traces, timeout) + else: + _, proto = await self._create_direct_connection(req, traces, timeout) + + return proto + + def _get_ssl_context(self, req: ClientRequest) -> Optional[SSLContext]: + """Logic to get the correct SSL context + + 0. if req.ssl is false, return None + + 1. if ssl_context is specified in req, use it + 2. if _ssl_context is specified in self, use it + 3. otherwise: + 1. if verify_ssl is not specified in req, use self.ssl_context + (will generate a default context according to self.verify_ssl) + 2. if verify_ssl is True in req, generate a default SSL context + 3. if verify_ssl is False in req, generate a SSL context that + won't verify + """ + if not req.is_ssl(): + return None + + if ssl is None: # pragma: no cover + raise RuntimeError("SSL is not supported.") + sslcontext = req.ssl + if isinstance(sslcontext, ssl.SSLContext): + return sslcontext + if sslcontext is not True: + # not verified or fingerprinted + return _SSL_CONTEXT_UNVERIFIED + sslcontext = self._ssl + if isinstance(sslcontext, ssl.SSLContext): + return sslcontext + if sslcontext is not True: + # not verified or fingerprinted + return _SSL_CONTEXT_UNVERIFIED + return _SSL_CONTEXT_VERIFIED + + def _get_fingerprint(self, req: ClientRequest) -> Optional["Fingerprint"]: + ret = req.ssl + if isinstance(ret, Fingerprint): + return ret + ret = self._ssl + if isinstance(ret, Fingerprint): + return ret + return None + + async def _wrap_create_connection( + self, + *args: Any, + addr_infos: List[AddrInfoType], + req: ClientRequest, + timeout: "ClientTimeout", + client_error: Type[Exception] = ClientConnectorError, + **kwargs: Any, + ) -> Tuple[asyncio.Transport, ResponseHandler]: + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + sock = await aiohappyeyeballs.start_connection( + addr_infos=addr_infos, + local_addr_infos=self._local_addr_infos, + happy_eyeballs_delay=self._happy_eyeballs_delay, + interleave=self._interleave, + loop=self._loop, + socket_factory=self._socket_factory, + ) + # Add ssl_shutdown_timeout for Python 3.11+ when SSL is used + if ( + kwargs.get("ssl") + and self._ssl_shutdown_timeout + and sys.version_info >= (3, 11) + ): + kwargs["ssl_shutdown_timeout"] = self._ssl_shutdown_timeout + return await self._loop.create_connection(*args, **kwargs, sock=sock) + except cert_errors as exc: + raise ClientConnectorCertificateError(req.connection_key, exc) from exc + except ssl_errors as exc: + raise ClientConnectorSSLError(req.connection_key, exc) from exc + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise client_error(req.connection_key, exc) from exc + + async def _wrap_existing_connection( + self, + *args: Any, + req: ClientRequest, + timeout: "ClientTimeout", + client_error: Type[Exception] = ClientConnectorError, + **kwargs: Any, + ) -> Tuple[asyncio.Transport, ResponseHandler]: + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + return await self._loop.create_connection(*args, **kwargs) + except cert_errors as exc: + raise ClientConnectorCertificateError(req.connection_key, exc) from exc + except ssl_errors as exc: + raise ClientConnectorSSLError(req.connection_key, exc) from exc + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise client_error(req.connection_key, exc) from exc + + def _fail_on_no_start_tls(self, req: "ClientRequest") -> None: + """Raise a :py:exc:`RuntimeError` on missing ``start_tls()``. + + It is necessary for TLS-in-TLS so that it is possible to + send HTTPS queries through HTTPS proxies. + + This doesn't affect regular HTTP requests, though. + """ + if not req.is_ssl(): + return + + proxy_url = req.proxy + assert proxy_url is not None + if proxy_url.scheme != "https": + return + + self._check_loop_for_start_tls() + + def _check_loop_for_start_tls(self) -> None: + try: + self._loop.start_tls + except AttributeError as attr_exc: + raise RuntimeError( + "An HTTPS request is being sent through an HTTPS proxy. " + "This needs support for TLS in TLS but it is not implemented " + "in your runtime for the stdlib asyncio.\n\n" + "Please upgrade to Python 3.11 or higher. For more details, " + "please see:\n" + "* https://bugs.python.org/issue37179\n" + "* https://github.com/python/cpython/pull/28073\n" + "* https://docs.aiohttp.org/en/stable/" + "client_advanced.html#proxy-support\n" + "* https://github.com/aio-libs/aiohttp/discussions/6044\n", + ) from attr_exc + + def _loop_supports_start_tls(self) -> bool: + try: + self._check_loop_for_start_tls() + except RuntimeError: + return False + else: + return True + + def _warn_about_tls_in_tls( + self, + underlying_transport: asyncio.Transport, + req: ClientRequest, + ) -> None: + """Issue a warning if the requested URL has HTTPS scheme.""" + if req.request_info.url.scheme != "https": + return + + # Check if uvloop is being used, which supports TLS in TLS, + # otherwise assume that asyncio's native transport is being used. + if type(underlying_transport).__module__.startswith("uvloop"): + return + + # Support in asyncio was added in Python 3.11 (bpo-44011) + asyncio_supports_tls_in_tls = sys.version_info >= (3, 11) or getattr( + underlying_transport, + "_start_tls_compatible", + False, + ) + + if asyncio_supports_tls_in_tls: + return + + warnings.warn( + "An HTTPS request is being sent through an HTTPS proxy. " + "This support for TLS in TLS is known to be disabled " + "in the stdlib asyncio (Python <3.11). This is why you'll probably see " + "an error in the log below.\n\n" + "It is possible to enable it via monkeypatching. " + "For more details, see:\n" + "* https://bugs.python.org/issue37179\n" + "* https://github.com/python/cpython/pull/28073\n\n" + "You can temporarily patch this as follows:\n" + "* https://docs.aiohttp.org/en/stable/client_advanced.html#proxy-support\n" + "* https://github.com/aio-libs/aiohttp/discussions/6044\n", + RuntimeWarning, + source=self, + # Why `4`? At least 3 of the calls in the stack originate + # from the methods in this class. + stacklevel=3, + ) + + async def _start_tls_connection( + self, + underlying_transport: asyncio.Transport, + req: ClientRequest, + timeout: "ClientTimeout", + client_error: Type[Exception] = ClientConnectorError, + ) -> Tuple[asyncio.BaseTransport, ResponseHandler]: + """Wrap the raw TCP transport with TLS.""" + tls_proto = self._factory() # Create a brand new proto for TLS + sslcontext = self._get_ssl_context(req) + if TYPE_CHECKING: + # _start_tls_connection is unreachable in the current code path + # if sslcontext is None. + assert sslcontext is not None + + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + try: + # ssl_shutdown_timeout is only available in Python 3.11+ + if sys.version_info >= (3, 11) and self._ssl_shutdown_timeout: + tls_transport = await self._loop.start_tls( + underlying_transport, + tls_proto, + sslcontext, + server_hostname=req.server_hostname or req.host, + ssl_handshake_timeout=timeout.total, + ssl_shutdown_timeout=self._ssl_shutdown_timeout, + ) + else: + tls_transport = await self._loop.start_tls( + underlying_transport, + tls_proto, + sslcontext, + server_hostname=req.server_hostname or req.host, + ssl_handshake_timeout=timeout.total, + ) + except BaseException: + # We need to close the underlying transport since + # `start_tls()` probably failed before it had a + # chance to do this: + if self._ssl_shutdown_timeout == 0: + underlying_transport.abort() + else: + underlying_transport.close() + raise + if isinstance(tls_transport, asyncio.Transport): + fingerprint = self._get_fingerprint(req) + if fingerprint: + try: + fingerprint.check(tls_transport) + except ServerFingerprintMismatch: + tls_transport.close() + if not self._cleanup_closed_disabled: + self._cleanup_closed_transports.append(tls_transport) + raise + except cert_errors as exc: + raise ClientConnectorCertificateError(req.connection_key, exc) from exc + except ssl_errors as exc: + raise ClientConnectorSSLError(req.connection_key, exc) from exc + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise client_error(req.connection_key, exc) from exc + except TypeError as type_err: + # Example cause looks like this: + # TypeError: transport is not supported by start_tls() + + raise ClientConnectionError( + "Cannot initialize a TLS-in-TLS connection to host " + f"{req.host!s}:{req.port:d} through an underlying connection " + f"to an HTTPS proxy {req.proxy!s} ssl:{req.ssl or 'default'} " + f"[{type_err!s}]" + ) from type_err + else: + if tls_transport is None: + msg = "Failed to start TLS (possibly caused by closing transport)" + raise client_error(req.connection_key, OSError(msg)) + tls_proto.connection_made( + tls_transport + ) # Kick the state machine of the new TLS protocol + + return tls_transport, tls_proto + + def _convert_hosts_to_addr_infos( + self, hosts: List[ResolveResult] + ) -> List[AddrInfoType]: + """Converts the list of hosts to a list of addr_infos. + + The list of hosts is the result of a DNS lookup. The list of + addr_infos is the result of a call to `socket.getaddrinfo()`. + """ + addr_infos: List[AddrInfoType] = [] + for hinfo in hosts: + host = hinfo["host"] + is_ipv6 = ":" in host + family = socket.AF_INET6 if is_ipv6 else socket.AF_INET + if self._family and self._family != family: + continue + addr = (host, hinfo["port"], 0, 0) if is_ipv6 else (host, hinfo["port"]) + addr_infos.append( + (family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", addr) + ) + return addr_infos + + async def _create_direct_connection( + self, + req: ClientRequest, + traces: List["Trace"], + timeout: "ClientTimeout", + *, + client_error: Type[Exception] = ClientConnectorError, + ) -> Tuple[asyncio.Transport, ResponseHandler]: + sslcontext = self._get_ssl_context(req) + fingerprint = self._get_fingerprint(req) + + host = req.url.raw_host + assert host is not None + # Replace multiple trailing dots with a single one. + # A trailing dot is only present for fully-qualified domain names. + # See https://github.com/aio-libs/aiohttp/pull/7364. + if host.endswith(".."): + host = host.rstrip(".") + "." + port = req.port + assert port is not None + try: + # Cancelling this lookup should not cancel the underlying lookup + # or else the cancel event will get broadcast to all the waiters + # across all connections. + hosts = await self._resolve_host(host, port, traces=traces) + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + # in case of proxy it is not ClientProxyConnectionError + # it is problem of resolving proxy ip itself + raise ClientConnectorDNSError(req.connection_key, exc) from exc + + last_exc: Optional[Exception] = None + addr_infos = self._convert_hosts_to_addr_infos(hosts) + while addr_infos: + # Strip trailing dots, certificates contain FQDN without dots. + # See https://github.com/aio-libs/aiohttp/issues/3636 + server_hostname = ( + (req.server_hostname or host).rstrip(".") if sslcontext else None + ) + + try: + transp, proto = await self._wrap_create_connection( + self._factory, + timeout=timeout, + ssl=sslcontext, + addr_infos=addr_infos, + server_hostname=server_hostname, + req=req, + client_error=client_error, + ) + except (ClientConnectorError, asyncio.TimeoutError) as exc: + last_exc = exc + aiohappyeyeballs.pop_addr_infos_interleave(addr_infos, self._interleave) + continue + + if req.is_ssl() and fingerprint: + try: + fingerprint.check(transp) + except ServerFingerprintMismatch as exc: + transp.close() + if not self._cleanup_closed_disabled: + self._cleanup_closed_transports.append(transp) + last_exc = exc + # Remove the bad peer from the list of addr_infos + sock: socket.socket = transp.get_extra_info("socket") + bad_peer = sock.getpeername() + aiohappyeyeballs.remove_addr_infos(addr_infos, bad_peer) + continue + + return transp, proto + else: + assert last_exc is not None + raise last_exc + + async def _create_proxy_connection( + self, req: ClientRequest, traces: List["Trace"], timeout: "ClientTimeout" + ) -> Tuple[asyncio.BaseTransport, ResponseHandler]: + self._fail_on_no_start_tls(req) + runtime_has_start_tls = self._loop_supports_start_tls() + + headers: Dict[str, str] = {} + if req.proxy_headers is not None: + headers = req.proxy_headers # type: ignore[assignment] + headers[hdrs.HOST] = req.headers[hdrs.HOST] + + url = req.proxy + assert url is not None + proxy_req = ClientRequest( + hdrs.METH_GET, + url, + headers=headers, + auth=req.proxy_auth, + loop=self._loop, + ssl=req.ssl, + ) + + # create connection to proxy server + transport, proto = await self._create_direct_connection( + proxy_req, [], timeout, client_error=ClientProxyConnectionError + ) + + auth = proxy_req.headers.pop(hdrs.AUTHORIZATION, None) + if auth is not None: + if not req.is_ssl(): + req.headers[hdrs.PROXY_AUTHORIZATION] = auth + else: + proxy_req.headers[hdrs.PROXY_AUTHORIZATION] = auth + + if req.is_ssl(): + if runtime_has_start_tls: + self._warn_about_tls_in_tls(transport, req) + + # For HTTPS requests over HTTP proxy + # we must notify proxy to tunnel connection + # so we send CONNECT command: + # CONNECT www.python.org:443 HTTP/1.1 + # Host: www.python.org + # + # next we must do TLS handshake and so on + # to do this we must wrap raw socket into secure one + # asyncio handles this perfectly + proxy_req.method = hdrs.METH_CONNECT + proxy_req.url = req.url + key = req.connection_key._replace( + proxy=None, proxy_auth=None, proxy_headers_hash=None + ) + conn = _ConnectTunnelConnection(self, key, proto, self._loop) + proxy_resp = await proxy_req.send(conn) + try: + protocol = conn._protocol + assert protocol is not None + + # read_until_eof=True will ensure the connection isn't closed + # once the response is received and processed allowing + # START_TLS to work on the connection below. + protocol.set_response_params( + read_until_eof=runtime_has_start_tls, + timeout_ceil_threshold=self._timeout_ceil_threshold, + ) + resp = await proxy_resp.start(conn) + except BaseException: + proxy_resp.close() + conn.close() + raise + else: + conn._protocol = None + try: + if resp.status != 200: + message = resp.reason + if message is None: + message = HTTPStatus(resp.status).phrase + raise ClientHttpProxyError( + proxy_resp.request_info, + resp.history, + status=resp.status, + message=message, + headers=resp.headers, + ) + if not runtime_has_start_tls: + rawsock = transport.get_extra_info("socket", default=None) + if rawsock is None: + raise RuntimeError( + "Transport does not expose socket instance" + ) + # Duplicate the socket, so now we can close proxy transport + rawsock = rawsock.dup() + except BaseException: + # It shouldn't be closed in `finally` because it's fed to + # `loop.start_tls()` and the docs say not to touch it after + # passing there. + transport.close() + raise + finally: + if not runtime_has_start_tls: + transport.close() + + if not runtime_has_start_tls: + # HTTP proxy with support for upgrade to HTTPS + sslcontext = self._get_ssl_context(req) + return await self._wrap_existing_connection( + self._factory, + timeout=timeout, + ssl=sslcontext, + sock=rawsock, + server_hostname=req.host, + req=req, + ) + + return await self._start_tls_connection( + # Access the old transport for the last time before it's + # closed and forgotten forever: + transport, + req=req, + timeout=timeout, + ) + finally: + proxy_resp.close() + + return transport, proto + + +class UnixConnector(BaseConnector): + """Unix socket connector. + + path - Unix socket path. + keepalive_timeout - (optional) Keep-alive timeout. + force_close - Set to True to force close and do reconnect + after each request (and between redirects). + limit - The total number of simultaneous connections. + limit_per_host - Number of simultaneous connections to one host. + loop - Optional event loop. + """ + + allowed_protocol_schema_set = HIGH_LEVEL_SCHEMA_SET | frozenset({"unix"}) + + def __init__( + self, + path: str, + force_close: bool = False, + keepalive_timeout: Union[object, float, None] = sentinel, + limit: int = 100, + limit_per_host: int = 0, + loop: Optional[asyncio.AbstractEventLoop] = None, + ) -> None: + super().__init__( + force_close=force_close, + keepalive_timeout=keepalive_timeout, + limit=limit, + limit_per_host=limit_per_host, + loop=loop, + ) + self._path = path + + @property + def path(self) -> str: + """Path to unix socket.""" + return self._path + + async def _create_connection( + self, req: ClientRequest, traces: List["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + _, proto = await self._loop.create_unix_connection( + self._factory, self._path + ) + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise UnixClientConnectorError(self.path, req.connection_key, exc) from exc + + return proto + + +class NamedPipeConnector(BaseConnector): + """Named pipe connector. + + Only supported by the proactor event loop. + See also: https://docs.python.org/3/library/asyncio-eventloop.html + + path - Windows named pipe path. + keepalive_timeout - (optional) Keep-alive timeout. + force_close - Set to True to force close and do reconnect + after each request (and between redirects). + limit - The total number of simultaneous connections. + limit_per_host - Number of simultaneous connections to one host. + loop - Optional event loop. + """ + + allowed_protocol_schema_set = HIGH_LEVEL_SCHEMA_SET | frozenset({"npipe"}) + + def __init__( + self, + path: str, + force_close: bool = False, + keepalive_timeout: Union[object, float, None] = sentinel, + limit: int = 100, + limit_per_host: int = 0, + loop: Optional[asyncio.AbstractEventLoop] = None, + ) -> None: + super().__init__( + force_close=force_close, + keepalive_timeout=keepalive_timeout, + limit=limit, + limit_per_host=limit_per_host, + loop=loop, + ) + if not isinstance( + self._loop, + asyncio.ProactorEventLoop, # type: ignore[attr-defined] + ): + raise RuntimeError( + "Named Pipes only available in proactor loop under windows" + ) + self._path = path + + @property + def path(self) -> str: + """Path to the named pipe.""" + return self._path + + async def _create_connection( + self, req: ClientRequest, traces: List["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + try: + async with ceil_timeout( + timeout.sock_connect, ceil_threshold=timeout.ceil_threshold + ): + _, proto = await self._loop.create_pipe_connection( # type: ignore[attr-defined] + self._factory, self._path + ) + # the drain is required so that the connection_made is called + # and transport is set otherwise it is not set before the + # `assert conn.transport is not None` + # in client.py's _request method + await asyncio.sleep(0) + # other option is to manually set transport like + # `proto.transport = trans` + except OSError as exc: + if exc.errno is None and isinstance(exc, asyncio.TimeoutError): + raise + raise ClientConnectorError(req.connection_key, exc) from exc + + return cast(ResponseHandler, proto) diff --git a/lib/python3.1/site-packages/aiohttp/cookiejar.py b/lib/python3.1/site-packages/aiohttp/cookiejar.py new file mode 100644 index 0000000000000000000000000000000000000000..193648d4309fc9252332c0c1241ab6aafbd2fe05 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/cookiejar.py @@ -0,0 +1,522 @@ +import asyncio +import calendar +import contextlib +import datetime +import heapq +import itertools +import os # noqa +import pathlib +import pickle +import re +import time +import warnings +from collections import defaultdict +from collections.abc import Mapping +from http.cookies import BaseCookie, Morsel, SimpleCookie +from typing import ( + DefaultDict, + Dict, + Iterable, + Iterator, + List, + Optional, + Set, + Tuple, + Union, +) + +from yarl import URL + +from ._cookie_helpers import preserve_morsel_with_coded_value +from .abc import AbstractCookieJar, ClearCookiePredicate +from .helpers import is_ip_address +from .typedefs import LooseCookies, PathLike, StrOrURL + +__all__ = ("CookieJar", "DummyCookieJar") + + +CookieItem = Union[str, "Morsel[str]"] + +# We cache these string methods here as their use is in performance critical code. +_FORMAT_PATH = "{}/{}".format +_FORMAT_DOMAIN_REVERSED = "{1}.{0}".format + +# The minimum number of scheduled cookie expirations before we start cleaning up +# the expiration heap. This is a performance optimization to avoid cleaning up the +# heap too often when there are only a few scheduled expirations. +_MIN_SCHEDULED_COOKIE_EXPIRATION = 100 +_SIMPLE_COOKIE = SimpleCookie() + + +class CookieJar(AbstractCookieJar): + """Implements cookie storage adhering to RFC 6265.""" + + DATE_TOKENS_RE = re.compile( + r"[\x09\x20-\x2F\x3B-\x40\x5B-\x60\x7B-\x7E]*" + r"(?P[\x00-\x08\x0A-\x1F\d:a-zA-Z\x7F-\xFF]+)" + ) + + DATE_HMS_TIME_RE = re.compile(r"(\d{1,2}):(\d{1,2}):(\d{1,2})") + + DATE_DAY_OF_MONTH_RE = re.compile(r"(\d{1,2})") + + DATE_MONTH_RE = re.compile( + "(jan)|(feb)|(mar)|(apr)|(may)|(jun)|(jul)|(aug)|(sep)|(oct)|(nov)|(dec)", + re.I, + ) + + DATE_YEAR_RE = re.compile(r"(\d{2,4})") + + # calendar.timegm() fails for timestamps after datetime.datetime.max + # Minus one as a loss of precision occurs when timestamp() is called. + MAX_TIME = ( + int(datetime.datetime.max.replace(tzinfo=datetime.timezone.utc).timestamp()) - 1 + ) + try: + calendar.timegm(time.gmtime(MAX_TIME)) + except (OSError, ValueError): + # Hit the maximum representable time on Windows + # https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/localtime-localtime32-localtime64 + # Throws ValueError on PyPy 3.9, OSError elsewhere + MAX_TIME = calendar.timegm((3000, 12, 31, 23, 59, 59, -1, -1, -1)) + except OverflowError: + # #4515: datetime.max may not be representable on 32-bit platforms + MAX_TIME = 2**31 - 1 + # Avoid minuses in the future, 3x faster + SUB_MAX_TIME = MAX_TIME - 1 + + def __init__( + self, + *, + unsafe: bool = False, + quote_cookie: bool = True, + treat_as_secure_origin: Union[StrOrURL, List[StrOrURL], None] = None, + loop: Optional[asyncio.AbstractEventLoop] = None, + ) -> None: + super().__init__(loop=loop) + self._cookies: DefaultDict[Tuple[str, str], SimpleCookie] = defaultdict( + SimpleCookie + ) + self._morsel_cache: DefaultDict[Tuple[str, str], Dict[str, Morsel[str]]] = ( + defaultdict(dict) + ) + self._host_only_cookies: Set[Tuple[str, str]] = set() + self._unsafe = unsafe + self._quote_cookie = quote_cookie + if treat_as_secure_origin is None: + treat_as_secure_origin = [] + elif isinstance(treat_as_secure_origin, URL): + treat_as_secure_origin = [treat_as_secure_origin.origin()] + elif isinstance(treat_as_secure_origin, str): + treat_as_secure_origin = [URL(treat_as_secure_origin).origin()] + else: + treat_as_secure_origin = [ + URL(url).origin() if isinstance(url, str) else url.origin() + for url in treat_as_secure_origin + ] + self._treat_as_secure_origin = treat_as_secure_origin + self._expire_heap: List[Tuple[float, Tuple[str, str, str]]] = [] + self._expirations: Dict[Tuple[str, str, str], float] = {} + + @property + def quote_cookie(self) -> bool: + return self._quote_cookie + + def save(self, file_path: PathLike) -> None: + file_path = pathlib.Path(file_path) + with file_path.open(mode="wb") as f: + pickle.dump(self._cookies, f, pickle.HIGHEST_PROTOCOL) + + def load(self, file_path: PathLike) -> None: + file_path = pathlib.Path(file_path) + with file_path.open(mode="rb") as f: + self._cookies = pickle.load(f) + + def clear(self, predicate: Optional[ClearCookiePredicate] = None) -> None: + if predicate is None: + self._expire_heap.clear() + self._cookies.clear() + self._morsel_cache.clear() + self._host_only_cookies.clear() + self._expirations.clear() + return + + now = time.time() + to_del = [ + key + for (domain, path), cookie in self._cookies.items() + for name, morsel in cookie.items() + if ( + (key := (domain, path, name)) in self._expirations + and self._expirations[key] <= now + ) + or predicate(morsel) + ] + if to_del: + self._delete_cookies(to_del) + + def clear_domain(self, domain: str) -> None: + self.clear(lambda x: self._is_domain_match(domain, x["domain"])) + + def __iter__(self) -> "Iterator[Morsel[str]]": + self._do_expiration() + for val in self._cookies.values(): + yield from val.values() + + def __len__(self) -> int: + """Return number of cookies. + + This function does not iterate self to avoid unnecessary expiration + checks. + """ + return sum(len(cookie.values()) for cookie in self._cookies.values()) + + def _do_expiration(self) -> None: + """Remove expired cookies.""" + if not (expire_heap_len := len(self._expire_heap)): + return + + # If the expiration heap grows larger than the number expirations + # times two, we clean it up to avoid keeping expired entries in + # the heap and consuming memory. We guard this with a minimum + # threshold to avoid cleaning up the heap too often when there are + # only a few scheduled expirations. + if ( + expire_heap_len > _MIN_SCHEDULED_COOKIE_EXPIRATION + and expire_heap_len > len(self._expirations) * 2 + ): + # Remove any expired entries from the expiration heap + # that do not match the expiration time in the expirations + # as it means the cookie has been re-added to the heap + # with a different expiration time. + self._expire_heap = [ + entry + for entry in self._expire_heap + if self._expirations.get(entry[1]) == entry[0] + ] + heapq.heapify(self._expire_heap) + + now = time.time() + to_del: List[Tuple[str, str, str]] = [] + # Find any expired cookies and add them to the to-delete list + while self._expire_heap: + when, cookie_key = self._expire_heap[0] + if when > now: + break + heapq.heappop(self._expire_heap) + # Check if the cookie hasn't been re-added to the heap + # with a different expiration time as it will be removed + # later when it reaches the top of the heap and its + # expiration time is met. + if self._expirations.get(cookie_key) == when: + to_del.append(cookie_key) + + if to_del: + self._delete_cookies(to_del) + + def _delete_cookies(self, to_del: List[Tuple[str, str, str]]) -> None: + for domain, path, name in to_del: + self._host_only_cookies.discard((domain, name)) + self._cookies[(domain, path)].pop(name, None) + self._morsel_cache[(domain, path)].pop(name, None) + self._expirations.pop((domain, path, name), None) + + def _expire_cookie(self, when: float, domain: str, path: str, name: str) -> None: + cookie_key = (domain, path, name) + if self._expirations.get(cookie_key) == when: + # Avoid adding duplicates to the heap + return + heapq.heappush(self._expire_heap, (when, cookie_key)) + self._expirations[cookie_key] = when + + def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None: + """Update cookies.""" + hostname = response_url.raw_host + + if not self._unsafe and is_ip_address(hostname): + # Don't accept cookies from IPs + return + + if isinstance(cookies, Mapping): + cookies = cookies.items() + + for name, cookie in cookies: + if not isinstance(cookie, Morsel): + tmp = SimpleCookie() + tmp[name] = cookie # type: ignore[assignment] + cookie = tmp[name] + + domain = cookie["domain"] + + # ignore domains with trailing dots + if domain and domain[-1] == ".": + domain = "" + del cookie["domain"] + + if not domain and hostname is not None: + # Set the cookie's domain to the response hostname + # and set its host-only-flag + self._host_only_cookies.add((hostname, name)) + domain = cookie["domain"] = hostname + + if domain and domain[0] == ".": + # Remove leading dot + domain = domain[1:] + cookie["domain"] = domain + + if hostname and not self._is_domain_match(domain, hostname): + # Setting cookies for different domains is not allowed + continue + + path = cookie["path"] + if not path or path[0] != "/": + # Set the cookie's path to the response path + path = response_url.path + if not path.startswith("/"): + path = "/" + else: + # Cut everything from the last slash to the end + path = "/" + path[1 : path.rfind("/")] + cookie["path"] = path + path = path.rstrip("/") + + if max_age := cookie["max-age"]: + try: + delta_seconds = int(max_age) + max_age_expiration = min(time.time() + delta_seconds, self.MAX_TIME) + self._expire_cookie(max_age_expiration, domain, path, name) + except ValueError: + cookie["max-age"] = "" + + elif expires := cookie["expires"]: + if expire_time := self._parse_date(expires): + self._expire_cookie(expire_time, domain, path, name) + else: + cookie["expires"] = "" + + key = (domain, path) + if self._cookies[key].get(name) != cookie: + # Don't blow away the cache if the same + # cookie gets set again + self._cookies[key][name] = cookie + self._morsel_cache[key].pop(name, None) + + self._do_expiration() + + def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]": + """Returns this jar's cookies filtered by their attributes.""" + # We always use BaseCookie now since all + # cookies set on on filtered are fully constructed + # Morsels, not just names and values. + filtered: BaseCookie[str] = BaseCookie() + if not self._cookies: + # Skip do_expiration() if there are no cookies. + return filtered + self._do_expiration() + if not self._cookies: + # Skip rest of function if no non-expired cookies. + return filtered + if type(request_url) is not URL: + warnings.warn( + "filter_cookies expects yarl.URL instances only," + f"and will stop working in 4.x, got {type(request_url)}", + DeprecationWarning, + stacklevel=2, + ) + request_url = URL(request_url) + hostname = request_url.raw_host or "" + + is_not_secure = request_url.scheme not in ("https", "wss") + if is_not_secure and self._treat_as_secure_origin: + request_origin = URL() + with contextlib.suppress(ValueError): + request_origin = request_url.origin() + is_not_secure = request_origin not in self._treat_as_secure_origin + + # Send shared cookie + key = ("", "") + for c in self._cookies[key].values(): + # Check cache first + if c.key in self._morsel_cache[key]: + filtered[c.key] = self._morsel_cache[key][c.key] + continue + + # Build and cache the morsel + mrsl_val = self._build_morsel(c) + self._morsel_cache[key][c.key] = mrsl_val + filtered[c.key] = mrsl_val + + if is_ip_address(hostname): + if not self._unsafe: + return filtered + domains: Iterable[str] = (hostname,) + else: + # Get all the subdomains that might match a cookie (e.g. "foo.bar.com", "bar.com", "com") + domains = itertools.accumulate( + reversed(hostname.split(".")), _FORMAT_DOMAIN_REVERSED + ) + + # Get all the path prefixes that might match a cookie (e.g. "", "/foo", "/foo/bar") + paths = itertools.accumulate(request_url.path.split("/"), _FORMAT_PATH) + # Create every combination of (domain, path) pairs. + pairs = itertools.product(domains, paths) + + path_len = len(request_url.path) + # Point 2: https://www.rfc-editor.org/rfc/rfc6265.html#section-5.4 + for p in pairs: + if p not in self._cookies: + continue + for name, cookie in self._cookies[p].items(): + domain = cookie["domain"] + + if (domain, name) in self._host_only_cookies and domain != hostname: + continue + + # Skip edge case when the cookie has a trailing slash but request doesn't. + if len(cookie["path"]) > path_len: + continue + + if is_not_secure and cookie["secure"]: + continue + + # We already built the Morsel so reuse it here + if name in self._morsel_cache[p]: + filtered[name] = self._morsel_cache[p][name] + continue + + # Build and cache the morsel + mrsl_val = self._build_morsel(cookie) + self._morsel_cache[p][name] = mrsl_val + filtered[name] = mrsl_val + + return filtered + + def _build_morsel(self, cookie: Morsel[str]) -> Morsel[str]: + """Build a morsel for sending, respecting quote_cookie setting.""" + if self._quote_cookie and cookie.coded_value and cookie.coded_value[0] == '"': + return preserve_morsel_with_coded_value(cookie) + morsel: Morsel[str] = Morsel() + if self._quote_cookie: + value, coded_value = _SIMPLE_COOKIE.value_encode(cookie.value) + else: + coded_value = value = cookie.value + # We use __setstate__ instead of the public set() API because it allows us to + # bypass validation and set already validated state. This is more stable than + # setting protected attributes directly and unlikely to change since it would + # break pickling. + morsel.__setstate__({"key": cookie.key, "value": value, "coded_value": coded_value}) # type: ignore[attr-defined] + return morsel + + @staticmethod + def _is_domain_match(domain: str, hostname: str) -> bool: + """Implements domain matching adhering to RFC 6265.""" + if hostname == domain: + return True + + if not hostname.endswith(domain): + return False + + non_matching = hostname[: -len(domain)] + + if not non_matching.endswith("."): + return False + + return not is_ip_address(hostname) + + @classmethod + def _parse_date(cls, date_str: str) -> Optional[int]: + """Implements date string parsing adhering to RFC 6265.""" + if not date_str: + return None + + found_time = False + found_day = False + found_month = False + found_year = False + + hour = minute = second = 0 + day = 0 + month = 0 + year = 0 + + for token_match in cls.DATE_TOKENS_RE.finditer(date_str): + + token = token_match.group("token") + + if not found_time: + time_match = cls.DATE_HMS_TIME_RE.match(token) + if time_match: + found_time = True + hour, minute, second = (int(s) for s in time_match.groups()) + continue + + if not found_day: + day_match = cls.DATE_DAY_OF_MONTH_RE.match(token) + if day_match: + found_day = True + day = int(day_match.group()) + continue + + if not found_month: + month_match = cls.DATE_MONTH_RE.match(token) + if month_match: + found_month = True + assert month_match.lastindex is not None + month = month_match.lastindex + continue + + if not found_year: + year_match = cls.DATE_YEAR_RE.match(token) + if year_match: + found_year = True + year = int(year_match.group()) + + if 70 <= year <= 99: + year += 1900 + elif 0 <= year <= 69: + year += 2000 + + if False in (found_day, found_month, found_year, found_time): + return None + + if not 1 <= day <= 31: + return None + + if year < 1601 or hour > 23 or minute > 59 or second > 59: + return None + + return calendar.timegm((year, month, day, hour, minute, second, -1, -1, -1)) + + +class DummyCookieJar(AbstractCookieJar): + """Implements a dummy cookie storage. + + It can be used with the ClientSession when no cookie processing is needed. + + """ + + def __init__(self, *, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: + super().__init__(loop=loop) + + def __iter__(self) -> "Iterator[Morsel[str]]": + while False: + yield None + + def __len__(self) -> int: + return 0 + + @property + def quote_cookie(self) -> bool: + return True + + def clear(self, predicate: Optional[ClearCookiePredicate] = None) -> None: + pass + + def clear_domain(self, domain: str) -> None: + pass + + def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None: + pass + + def filter_cookies(self, request_url: URL) -> "BaseCookie[str]": + return SimpleCookie() diff --git a/lib/python3.1/site-packages/aiohttp/formdata.py b/lib/python3.1/site-packages/aiohttp/formdata.py new file mode 100644 index 0000000000000000000000000000000000000000..bdf591fae7ae05be98c223890282ffa57de69176 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/formdata.py @@ -0,0 +1,179 @@ +import io +import warnings +from typing import Any, Iterable, List, Optional +from urllib.parse import urlencode + +from multidict import MultiDict, MultiDictProxy + +from . import hdrs, multipart, payload +from .helpers import guess_filename +from .payload import Payload + +__all__ = ("FormData",) + + +class FormData: + """Helper class for form body generation. + + Supports multipart/form-data and application/x-www-form-urlencoded. + """ + + def __init__( + self, + fields: Iterable[Any] = (), + quote_fields: bool = True, + charset: Optional[str] = None, + *, + default_to_multipart: bool = False, + ) -> None: + self._writer = multipart.MultipartWriter("form-data") + self._fields: List[Any] = [] + self._is_multipart = default_to_multipart + self._quote_fields = quote_fields + self._charset = charset + + if isinstance(fields, dict): + fields = list(fields.items()) + elif not isinstance(fields, (list, tuple)): + fields = (fields,) + self.add_fields(*fields) + + @property + def is_multipart(self) -> bool: + return self._is_multipart + + def add_field( + self, + name: str, + value: Any, + *, + content_type: Optional[str] = None, + filename: Optional[str] = None, + content_transfer_encoding: Optional[str] = None, + ) -> None: + + if isinstance(value, io.IOBase): + self._is_multipart = True + elif isinstance(value, (bytes, bytearray, memoryview)): + msg = ( + "In v4, passing bytes will no longer create a file field. " + "Please explicitly use the filename parameter or pass a BytesIO object." + ) + if filename is None and content_transfer_encoding is None: + warnings.warn(msg, DeprecationWarning) + filename = name + + type_options: MultiDict[str] = MultiDict({"name": name}) + if filename is not None and not isinstance(filename, str): + raise TypeError("filename must be an instance of str. Got: %s" % filename) + if filename is None and isinstance(value, io.IOBase): + filename = guess_filename(value, name) + if filename is not None: + type_options["filename"] = filename + self._is_multipart = True + + headers = {} + if content_type is not None: + if not isinstance(content_type, str): + raise TypeError( + "content_type must be an instance of str. Got: %s" % content_type + ) + headers[hdrs.CONTENT_TYPE] = content_type + self._is_multipart = True + if content_transfer_encoding is not None: + if not isinstance(content_transfer_encoding, str): + raise TypeError( + "content_transfer_encoding must be an instance" + " of str. Got: %s" % content_transfer_encoding + ) + msg = ( + "content_transfer_encoding is deprecated. " + "To maintain compatibility with v4 please pass a BytesPayload." + ) + warnings.warn(msg, DeprecationWarning) + self._is_multipart = True + + self._fields.append((type_options, headers, value)) + + def add_fields(self, *fields: Any) -> None: + to_add = list(fields) + + while to_add: + rec = to_add.pop(0) + + if isinstance(rec, io.IOBase): + k = guess_filename(rec, "unknown") + self.add_field(k, rec) # type: ignore[arg-type] + + elif isinstance(rec, (MultiDictProxy, MultiDict)): + to_add.extend(rec.items()) + + elif isinstance(rec, (list, tuple)) and len(rec) == 2: + k, fp = rec + self.add_field(k, fp) # type: ignore[arg-type] + + else: + raise TypeError( + "Only io.IOBase, multidict and (name, file) " + "pairs allowed, use .add_field() for passing " + "more complex parameters, got {!r}".format(rec) + ) + + def _gen_form_urlencoded(self) -> payload.BytesPayload: + # form data (x-www-form-urlencoded) + data = [] + for type_options, _, value in self._fields: + data.append((type_options["name"], value)) + + charset = self._charset if self._charset is not None else "utf-8" + + if charset == "utf-8": + content_type = "application/x-www-form-urlencoded" + else: + content_type = "application/x-www-form-urlencoded; charset=%s" % charset + + return payload.BytesPayload( + urlencode(data, doseq=True, encoding=charset).encode(), + content_type=content_type, + ) + + def _gen_form_data(self) -> multipart.MultipartWriter: + """Encode a list of fields using the multipart/form-data MIME format""" + for dispparams, headers, value in self._fields: + try: + if hdrs.CONTENT_TYPE in headers: + part = payload.get_payload( + value, + content_type=headers[hdrs.CONTENT_TYPE], + headers=headers, + encoding=self._charset, + ) + else: + part = payload.get_payload( + value, headers=headers, encoding=self._charset + ) + except Exception as exc: + raise TypeError( + "Can not serialize value type: %r\n " + "headers: %r\n value: %r" % (type(value), headers, value) + ) from exc + + if dispparams: + part.set_content_disposition( + "form-data", quote_fields=self._quote_fields, **dispparams + ) + # FIXME cgi.FieldStorage doesn't likes body parts with + # Content-Length which were sent via chunked transfer encoding + assert part.headers is not None + part.headers.popall(hdrs.CONTENT_LENGTH, None) + + self._writer.append_payload(part) + + self._fields.clear() + return self._writer + + def __call__(self) -> Payload: + if self._is_multipart: + return self._gen_form_data() + else: + return self._gen_form_urlencoded() diff --git a/lib/python3.1/site-packages/aiohttp/hdrs.py b/lib/python3.1/site-packages/aiohttp/hdrs.py new file mode 100644 index 0000000000000000000000000000000000000000..c8d6b35f33ae4be537d7776ce4085982618d1305 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/hdrs.py @@ -0,0 +1,121 @@ +"""HTTP Headers constants.""" + +# After changing the file content call ./tools/gen.py +# to regenerate the headers parser +import itertools +from typing import Final, Set + +from multidict import istr + +METH_ANY: Final[str] = "*" +METH_CONNECT: Final[str] = "CONNECT" +METH_HEAD: Final[str] = "HEAD" +METH_GET: Final[str] = "GET" +METH_DELETE: Final[str] = "DELETE" +METH_OPTIONS: Final[str] = "OPTIONS" +METH_PATCH: Final[str] = "PATCH" +METH_POST: Final[str] = "POST" +METH_PUT: Final[str] = "PUT" +METH_TRACE: Final[str] = "TRACE" + +METH_ALL: Final[Set[str]] = { + METH_CONNECT, + METH_HEAD, + METH_GET, + METH_DELETE, + METH_OPTIONS, + METH_PATCH, + METH_POST, + METH_PUT, + METH_TRACE, +} + +ACCEPT: Final[istr] = istr("Accept") +ACCEPT_CHARSET: Final[istr] = istr("Accept-Charset") +ACCEPT_ENCODING: Final[istr] = istr("Accept-Encoding") +ACCEPT_LANGUAGE: Final[istr] = istr("Accept-Language") +ACCEPT_RANGES: Final[istr] = istr("Accept-Ranges") +ACCESS_CONTROL_MAX_AGE: Final[istr] = istr("Access-Control-Max-Age") +ACCESS_CONTROL_ALLOW_CREDENTIALS: Final[istr] = istr("Access-Control-Allow-Credentials") +ACCESS_CONTROL_ALLOW_HEADERS: Final[istr] = istr("Access-Control-Allow-Headers") +ACCESS_CONTROL_ALLOW_METHODS: Final[istr] = istr("Access-Control-Allow-Methods") +ACCESS_CONTROL_ALLOW_ORIGIN: Final[istr] = istr("Access-Control-Allow-Origin") +ACCESS_CONTROL_EXPOSE_HEADERS: Final[istr] = istr("Access-Control-Expose-Headers") +ACCESS_CONTROL_REQUEST_HEADERS: Final[istr] = istr("Access-Control-Request-Headers") +ACCESS_CONTROL_REQUEST_METHOD: Final[istr] = istr("Access-Control-Request-Method") +AGE: Final[istr] = istr("Age") +ALLOW: Final[istr] = istr("Allow") +AUTHORIZATION: Final[istr] = istr("Authorization") +CACHE_CONTROL: Final[istr] = istr("Cache-Control") +CONNECTION: Final[istr] = istr("Connection") +CONTENT_DISPOSITION: Final[istr] = istr("Content-Disposition") +CONTENT_ENCODING: Final[istr] = istr("Content-Encoding") +CONTENT_LANGUAGE: Final[istr] = istr("Content-Language") +CONTENT_LENGTH: Final[istr] = istr("Content-Length") +CONTENT_LOCATION: Final[istr] = istr("Content-Location") +CONTENT_MD5: Final[istr] = istr("Content-MD5") +CONTENT_RANGE: Final[istr] = istr("Content-Range") +CONTENT_TRANSFER_ENCODING: Final[istr] = istr("Content-Transfer-Encoding") +CONTENT_TYPE: Final[istr] = istr("Content-Type") +COOKIE: Final[istr] = istr("Cookie") +DATE: Final[istr] = istr("Date") +DESTINATION: Final[istr] = istr("Destination") +DIGEST: Final[istr] = istr("Digest") +ETAG: Final[istr] = istr("Etag") +EXPECT: Final[istr] = istr("Expect") +EXPIRES: Final[istr] = istr("Expires") +FORWARDED: Final[istr] = istr("Forwarded") +FROM: Final[istr] = istr("From") +HOST: Final[istr] = istr("Host") +IF_MATCH: Final[istr] = istr("If-Match") +IF_MODIFIED_SINCE: Final[istr] = istr("If-Modified-Since") +IF_NONE_MATCH: Final[istr] = istr("If-None-Match") +IF_RANGE: Final[istr] = istr("If-Range") +IF_UNMODIFIED_SINCE: Final[istr] = istr("If-Unmodified-Since") +KEEP_ALIVE: Final[istr] = istr("Keep-Alive") +LAST_EVENT_ID: Final[istr] = istr("Last-Event-ID") +LAST_MODIFIED: Final[istr] = istr("Last-Modified") +LINK: Final[istr] = istr("Link") +LOCATION: Final[istr] = istr("Location") +MAX_FORWARDS: Final[istr] = istr("Max-Forwards") +ORIGIN: Final[istr] = istr("Origin") +PRAGMA: Final[istr] = istr("Pragma") +PROXY_AUTHENTICATE: Final[istr] = istr("Proxy-Authenticate") +PROXY_AUTHORIZATION: Final[istr] = istr("Proxy-Authorization") +RANGE: Final[istr] = istr("Range") +REFERER: Final[istr] = istr("Referer") +RETRY_AFTER: Final[istr] = istr("Retry-After") +SEC_WEBSOCKET_ACCEPT: Final[istr] = istr("Sec-WebSocket-Accept") +SEC_WEBSOCKET_VERSION: Final[istr] = istr("Sec-WebSocket-Version") +SEC_WEBSOCKET_PROTOCOL: Final[istr] = istr("Sec-WebSocket-Protocol") +SEC_WEBSOCKET_EXTENSIONS: Final[istr] = istr("Sec-WebSocket-Extensions") +SEC_WEBSOCKET_KEY: Final[istr] = istr("Sec-WebSocket-Key") +SEC_WEBSOCKET_KEY1: Final[istr] = istr("Sec-WebSocket-Key1") +SERVER: Final[istr] = istr("Server") +SET_COOKIE: Final[istr] = istr("Set-Cookie") +TE: Final[istr] = istr("TE") +TRAILER: Final[istr] = istr("Trailer") +TRANSFER_ENCODING: Final[istr] = istr("Transfer-Encoding") +UPGRADE: Final[istr] = istr("Upgrade") +URI: Final[istr] = istr("URI") +USER_AGENT: Final[istr] = istr("User-Agent") +VARY: Final[istr] = istr("Vary") +VIA: Final[istr] = istr("Via") +WANT_DIGEST: Final[istr] = istr("Want-Digest") +WARNING: Final[istr] = istr("Warning") +WWW_AUTHENTICATE: Final[istr] = istr("WWW-Authenticate") +X_FORWARDED_FOR: Final[istr] = istr("X-Forwarded-For") +X_FORWARDED_HOST: Final[istr] = istr("X-Forwarded-Host") +X_FORWARDED_PROTO: Final[istr] = istr("X-Forwarded-Proto") + +# These are the upper/lower case variants of the headers/methods +# Example: {'hOst', 'host', 'HoST', 'HOSt', 'hOsT', 'HosT', 'hoSt', ...} +METH_HEAD_ALL: Final = frozenset( + map("".join, itertools.product(*zip(METH_HEAD.upper(), METH_HEAD.lower()))) +) +METH_CONNECT_ALL: Final = frozenset( + map("".join, itertools.product(*zip(METH_CONNECT.upper(), METH_CONNECT.lower()))) +) +HOST_ALL: Final = frozenset( + map("".join, itertools.product(*zip(HOST.upper(), HOST.lower()))) +) diff --git a/lib/python3.1/site-packages/aiohttp/helpers.py b/lib/python3.1/site-packages/aiohttp/helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..ace4f0e9b5305dd68b6683c68a1627c037d809ae --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/helpers.py @@ -0,0 +1,958 @@ +"""Various helper functions""" + +import asyncio +import base64 +import binascii +import contextlib +import datetime +import enum +import functools +import inspect +import netrc +import os +import platform +import re +import sys +import time +import weakref +from collections import namedtuple +from contextlib import suppress +from email.parser import HeaderParser +from email.utils import parsedate +from math import ceil +from pathlib import Path +from types import MappingProxyType, TracebackType +from typing import ( + Any, + Callable, + ContextManager, + Dict, + Generator, + Generic, + Iterable, + Iterator, + List, + Mapping, + Optional, + Protocol, + Tuple, + Type, + TypeVar, + Union, + get_args, + overload, +) +from urllib.parse import quote +from urllib.request import getproxies, proxy_bypass + +import attr +from multidict import MultiDict, MultiDictProxy, MultiMapping +from propcache.api import under_cached_property as reify +from yarl import URL + +from . import hdrs +from .log import client_logger + +if sys.version_info >= (3, 11): + import asyncio as async_timeout +else: + import async_timeout + +__all__ = ("BasicAuth", "ChainMapProxy", "ETag", "reify") + +IS_MACOS = platform.system() == "Darwin" +IS_WINDOWS = platform.system() == "Windows" + +PY_310 = sys.version_info >= (3, 10) +PY_311 = sys.version_info >= (3, 11) + + +_T = TypeVar("_T") +_S = TypeVar("_S") + +_SENTINEL = enum.Enum("_SENTINEL", "sentinel") +sentinel = _SENTINEL.sentinel + +NO_EXTENSIONS = bool(os.environ.get("AIOHTTP_NO_EXTENSIONS")) + +# https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.1 +EMPTY_BODY_STATUS_CODES = frozenset((204, 304, *range(100, 200))) +# https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.1 +# https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.2 +EMPTY_BODY_METHODS = hdrs.METH_HEAD_ALL + +DEBUG = sys.flags.dev_mode or ( + not sys.flags.ignore_environment and bool(os.environ.get("PYTHONASYNCIODEBUG")) +) + + +CHAR = {chr(i) for i in range(0, 128)} +CTL = {chr(i) for i in range(0, 32)} | { + chr(127), +} +SEPARATORS = { + "(", + ")", + "<", + ">", + "@", + ",", + ";", + ":", + "\\", + '"', + "/", + "[", + "]", + "?", + "=", + "{", + "}", + " ", + chr(9), +} +TOKEN = CHAR ^ CTL ^ SEPARATORS + + +class noop: + def __await__(self) -> Generator[None, None, None]: + yield + + +class BasicAuth(namedtuple("BasicAuth", ["login", "password", "encoding"])): + """Http basic authentication helper.""" + + def __new__( + cls, login: str, password: str = "", encoding: str = "latin1" + ) -> "BasicAuth": + if login is None: + raise ValueError("None is not allowed as login value") + + if password is None: + raise ValueError("None is not allowed as password value") + + if ":" in login: + raise ValueError('A ":" is not allowed in login (RFC 1945#section-11.1)') + + return super().__new__(cls, login, password, encoding) + + @classmethod + def decode(cls, auth_header: str, encoding: str = "latin1") -> "BasicAuth": + """Create a BasicAuth object from an Authorization HTTP header.""" + try: + auth_type, encoded_credentials = auth_header.split(" ", 1) + except ValueError: + raise ValueError("Could not parse authorization header.") + + if auth_type.lower() != "basic": + raise ValueError("Unknown authorization method %s" % auth_type) + + try: + decoded = base64.b64decode( + encoded_credentials.encode("ascii"), validate=True + ).decode(encoding) + except binascii.Error: + raise ValueError("Invalid base64 encoding.") + + try: + # RFC 2617 HTTP Authentication + # https://www.ietf.org/rfc/rfc2617.txt + # the colon must be present, but the username and password may be + # otherwise blank. + username, password = decoded.split(":", 1) + except ValueError: + raise ValueError("Invalid credentials.") + + return cls(username, password, encoding=encoding) + + @classmethod + def from_url(cls, url: URL, *, encoding: str = "latin1") -> Optional["BasicAuth"]: + """Create BasicAuth from url.""" + if not isinstance(url, URL): + raise TypeError("url should be yarl.URL instance") + # Check raw_user and raw_password first as yarl is likely + # to already have these values parsed from the netloc in the cache. + if url.raw_user is None and url.raw_password is None: + return None + return cls(url.user or "", url.password or "", encoding=encoding) + + def encode(self) -> str: + """Encode credentials.""" + creds = (f"{self.login}:{self.password}").encode(self.encoding) + return "Basic %s" % base64.b64encode(creds).decode(self.encoding) + + +def strip_auth_from_url(url: URL) -> Tuple[URL, Optional[BasicAuth]]: + """Remove user and password from URL if present and return BasicAuth object.""" + # Check raw_user and raw_password first as yarl is likely + # to already have these values parsed from the netloc in the cache. + if url.raw_user is None and url.raw_password is None: + return url, None + return url.with_user(None), BasicAuth(url.user or "", url.password or "") + + +def netrc_from_env() -> Optional[netrc.netrc]: + """Load netrc from file. + + Attempt to load it from the path specified by the env-var + NETRC or in the default location in the user's home directory. + + Returns None if it couldn't be found or fails to parse. + """ + netrc_env = os.environ.get("NETRC") + + if netrc_env is not None: + netrc_path = Path(netrc_env) + else: + try: + home_dir = Path.home() + except RuntimeError as e: # pragma: no cover + # if pathlib can't resolve home, it may raise a RuntimeError + client_logger.debug( + "Could not resolve home directory when " + "trying to look for .netrc file: %s", + e, + ) + return None + + netrc_path = home_dir / ("_netrc" if IS_WINDOWS else ".netrc") + + try: + return netrc.netrc(str(netrc_path)) + except netrc.NetrcParseError as e: + client_logger.warning("Could not parse .netrc file: %s", e) + except OSError as e: + netrc_exists = False + with contextlib.suppress(OSError): + netrc_exists = netrc_path.is_file() + # we couldn't read the file (doesn't exist, permissions, etc.) + if netrc_env or netrc_exists: + # only warn if the environment wanted us to load it, + # or it appears like the default file does actually exist + client_logger.warning("Could not read .netrc file: %s", e) + + return None + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ProxyInfo: + proxy: URL + proxy_auth: Optional[BasicAuth] + + +def basicauth_from_netrc(netrc_obj: Optional[netrc.netrc], host: str) -> BasicAuth: + """ + Return :py:class:`~aiohttp.BasicAuth` credentials for ``host`` from ``netrc_obj``. + + :raises LookupError: if ``netrc_obj`` is :py:data:`None` or if no + entry is found for the ``host``. + """ + if netrc_obj is None: + raise LookupError("No .netrc file found") + auth_from_netrc = netrc_obj.authenticators(host) + + if auth_from_netrc is None: + raise LookupError(f"No entry for {host!s} found in the `.netrc` file.") + login, account, password = auth_from_netrc + + # TODO(PY311): username = login or account + # Up to python 3.10, account could be None if not specified, + # and login will be empty string if not specified. From 3.11, + # login and account will be empty string if not specified. + username = login if (login or account is None) else account + + # TODO(PY311): Remove this, as password will be empty string + # if not specified + if password is None: + password = "" + + return BasicAuth(username, password) + + +def proxies_from_env() -> Dict[str, ProxyInfo]: + proxy_urls = { + k: URL(v) + for k, v in getproxies().items() + if k in ("http", "https", "ws", "wss") + } + netrc_obj = netrc_from_env() + stripped = {k: strip_auth_from_url(v) for k, v in proxy_urls.items()} + ret = {} + for proto, val in stripped.items(): + proxy, auth = val + if proxy.scheme in ("https", "wss"): + client_logger.warning( + "%s proxies %s are not supported, ignoring", proxy.scheme.upper(), proxy + ) + continue + if netrc_obj and auth is None: + if proxy.host is not None: + try: + auth = basicauth_from_netrc(netrc_obj, proxy.host) + except LookupError: + auth = None + ret[proto] = ProxyInfo(proxy, auth) + return ret + + +def get_env_proxy_for_url(url: URL) -> Tuple[URL, Optional[BasicAuth]]: + """Get a permitted proxy for the given URL from the env.""" + if url.host is not None and proxy_bypass(url.host): + raise LookupError(f"Proxying is disallowed for `{url.host!r}`") + + proxies_in_env = proxies_from_env() + try: + proxy_info = proxies_in_env[url.scheme] + except KeyError: + raise LookupError(f"No proxies found for `{url!s}` in the env") + else: + return proxy_info.proxy, proxy_info.proxy_auth + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class MimeType: + type: str + subtype: str + suffix: str + parameters: "MultiDictProxy[str]" + + +@functools.lru_cache(maxsize=56) +def parse_mimetype(mimetype: str) -> MimeType: + """Parses a MIME type into its components. + + mimetype is a MIME type string. + + Returns a MimeType object. + + Example: + + >>> parse_mimetype('text/html; charset=utf-8') + MimeType(type='text', subtype='html', suffix='', + parameters={'charset': 'utf-8'}) + + """ + if not mimetype: + return MimeType( + type="", subtype="", suffix="", parameters=MultiDictProxy(MultiDict()) + ) + + parts = mimetype.split(";") + params: MultiDict[str] = MultiDict() + for item in parts[1:]: + if not item: + continue + key, _, value = item.partition("=") + params.add(key.lower().strip(), value.strip(' "')) + + fulltype = parts[0].strip().lower() + if fulltype == "*": + fulltype = "*/*" + + mtype, _, stype = fulltype.partition("/") + stype, _, suffix = stype.partition("+") + + return MimeType( + type=mtype, subtype=stype, suffix=suffix, parameters=MultiDictProxy(params) + ) + + +@functools.lru_cache(maxsize=56) +def parse_content_type(raw: str) -> Tuple[str, MappingProxyType[str, str]]: + """Parse Content-Type header. + + Returns a tuple of the parsed content type and a + MappingProxyType of parameters. + """ + msg = HeaderParser().parsestr(f"Content-Type: {raw}") + content_type = msg.get_content_type() + params = msg.get_params(()) + content_dict = dict(params[1:]) # First element is content type again + return content_type, MappingProxyType(content_dict) + + +def guess_filename(obj: Any, default: Optional[str] = None) -> Optional[str]: + name = getattr(obj, "name", None) + if name and isinstance(name, str) and name[0] != "<" and name[-1] != ">": + return Path(name).name + return default + + +not_qtext_re = re.compile(r"[^\041\043-\133\135-\176]") +QCONTENT = {chr(i) for i in range(0x20, 0x7F)} | {"\t"} + + +def quoted_string(content: str) -> str: + """Return 7-bit content as quoted-string. + + Format content into a quoted-string as defined in RFC5322 for + Internet Message Format. Notice that this is not the 8-bit HTTP + format, but the 7-bit email format. Content must be in usascii or + a ValueError is raised. + """ + if not (QCONTENT > set(content)): + raise ValueError(f"bad content for quoted-string {content!r}") + return not_qtext_re.sub(lambda x: "\\" + x.group(0), content) + + +def content_disposition_header( + disptype: str, quote_fields: bool = True, _charset: str = "utf-8", **params: str +) -> str: + """Sets ``Content-Disposition`` header for MIME. + + This is the MIME payload Content-Disposition header from RFC 2183 + and RFC 7579 section 4.2, not the HTTP Content-Disposition from + RFC 6266. + + disptype is a disposition type: inline, attachment, form-data. + Should be valid extension token (see RFC 2183) + + quote_fields performs value quoting to 7-bit MIME headers + according to RFC 7578. Set to quote_fields to False if recipient + can take 8-bit file names and field values. + + _charset specifies the charset to use when quote_fields is True. + + params is a dict with disposition params. + """ + if not disptype or not (TOKEN > set(disptype)): + raise ValueError(f"bad content disposition type {disptype!r}") + + value = disptype + if params: + lparams = [] + for key, val in params.items(): + if not key or not (TOKEN > set(key)): + raise ValueError(f"bad content disposition parameter {key!r}={val!r}") + if quote_fields: + if key.lower() == "filename": + qval = quote(val, "", encoding=_charset) + lparams.append((key, '"%s"' % qval)) + else: + try: + qval = quoted_string(val) + except ValueError: + qval = "".join( + (_charset, "''", quote(val, "", encoding=_charset)) + ) + lparams.append((key + "*", qval)) + else: + lparams.append((key, '"%s"' % qval)) + else: + qval = val.replace("\\", "\\\\").replace('"', '\\"') + lparams.append((key, '"%s"' % qval)) + sparams = "; ".join("=".join(pair) for pair in lparams) + value = "; ".join((value, sparams)) + return value + + +def is_ip_address(host: Optional[str]) -> bool: + """Check if host looks like an IP Address. + + This check is only meant as a heuristic to ensure that + a host is not a domain name. + """ + if not host: + return False + # For a host to be an ipv4 address, it must be all numeric. + # The host must contain a colon to be an IPv6 address. + return ":" in host or host.replace(".", "").isdigit() + + +_cached_current_datetime: Optional[int] = None +_cached_formatted_datetime = "" + + +def rfc822_formatted_time() -> str: + global _cached_current_datetime + global _cached_formatted_datetime + + now = int(time.time()) + if now != _cached_current_datetime: + # Weekday and month names for HTTP date/time formatting; + # always English! + # Tuples are constants stored in codeobject! + _weekdayname = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun") + _monthname = ( + "", # Dummy so we can use 1-based month numbers + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", + ) + + year, month, day, hh, mm, ss, wd, *tail = time.gmtime(now) + _cached_formatted_datetime = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( + _weekdayname[wd], + day, + _monthname[month], + year, + hh, + mm, + ss, + ) + _cached_current_datetime = now + return _cached_formatted_datetime + + +def _weakref_handle(info: "Tuple[weakref.ref[object], str]") -> None: + ref, name = info + ob = ref() + if ob is not None: + with suppress(Exception): + getattr(ob, name)() + + +def weakref_handle( + ob: object, + name: str, + timeout: float, + loop: asyncio.AbstractEventLoop, + timeout_ceil_threshold: float = 5, +) -> Optional[asyncio.TimerHandle]: + if timeout is not None and timeout > 0: + when = loop.time() + timeout + if timeout >= timeout_ceil_threshold: + when = ceil(when) + + return loop.call_at(when, _weakref_handle, (weakref.ref(ob), name)) + return None + + +def call_later( + cb: Callable[[], Any], + timeout: float, + loop: asyncio.AbstractEventLoop, + timeout_ceil_threshold: float = 5, +) -> Optional[asyncio.TimerHandle]: + if timeout is None or timeout <= 0: + return None + now = loop.time() + when = calculate_timeout_when(now, timeout, timeout_ceil_threshold) + return loop.call_at(when, cb) + + +def calculate_timeout_when( + loop_time: float, + timeout: float, + timeout_ceiling_threshold: float, +) -> float: + """Calculate when to execute a timeout.""" + when = loop_time + timeout + if timeout > timeout_ceiling_threshold: + return ceil(when) + return when + + +class TimeoutHandle: + """Timeout handle""" + + __slots__ = ("_timeout", "_loop", "_ceil_threshold", "_callbacks") + + def __init__( + self, + loop: asyncio.AbstractEventLoop, + timeout: Optional[float], + ceil_threshold: float = 5, + ) -> None: + self._timeout = timeout + self._loop = loop + self._ceil_threshold = ceil_threshold + self._callbacks: List[ + Tuple[Callable[..., None], Tuple[Any, ...], Dict[str, Any]] + ] = [] + + def register( + self, callback: Callable[..., None], *args: Any, **kwargs: Any + ) -> None: + self._callbacks.append((callback, args, kwargs)) + + def close(self) -> None: + self._callbacks.clear() + + def start(self) -> Optional[asyncio.TimerHandle]: + timeout = self._timeout + if timeout is not None and timeout > 0: + when = self._loop.time() + timeout + if timeout >= self._ceil_threshold: + when = ceil(when) + return self._loop.call_at(when, self.__call__) + else: + return None + + def timer(self) -> "BaseTimerContext": + if self._timeout is not None and self._timeout > 0: + timer = TimerContext(self._loop) + self.register(timer.timeout) + return timer + else: + return TimerNoop() + + def __call__(self) -> None: + for cb, args, kwargs in self._callbacks: + with suppress(Exception): + cb(*args, **kwargs) + + self._callbacks.clear() + + +class BaseTimerContext(ContextManager["BaseTimerContext"]): + + __slots__ = () + + def assert_timeout(self) -> None: + """Raise TimeoutError if timeout has been exceeded.""" + + +class TimerNoop(BaseTimerContext): + + __slots__ = () + + def __enter__(self) -> BaseTimerContext: + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + return + + +class TimerContext(BaseTimerContext): + """Low resolution timeout context manager""" + + __slots__ = ("_loop", "_tasks", "_cancelled", "_cancelling") + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + self._tasks: List[asyncio.Task[Any]] = [] + self._cancelled = False + self._cancelling = 0 + + def assert_timeout(self) -> None: + """Raise TimeoutError if timer has already been cancelled.""" + if self._cancelled: + raise asyncio.TimeoutError from None + + def __enter__(self) -> BaseTimerContext: + task = asyncio.current_task(loop=self._loop) + if task is None: + raise RuntimeError("Timeout context manager should be used inside a task") + + if sys.version_info >= (3, 11): + # Remember if the task was already cancelling + # so when we __exit__ we can decide if we should + # raise asyncio.TimeoutError or let the cancellation propagate + self._cancelling = task.cancelling() + + if self._cancelled: + raise asyncio.TimeoutError from None + + self._tasks.append(task) + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + enter_task: Optional[asyncio.Task[Any]] = None + if self._tasks: + enter_task = self._tasks.pop() + + if exc_type is asyncio.CancelledError and self._cancelled: + assert enter_task is not None + # The timeout was hit, and the task was cancelled + # so we need to uncancel the last task that entered the context manager + # since the cancellation should not leak out of the context manager + if sys.version_info >= (3, 11): + # If the task was already cancelling don't raise + # asyncio.TimeoutError and instead return None + # to allow the cancellation to propagate + if enter_task.uncancel() > self._cancelling: + return None + raise asyncio.TimeoutError from exc_val + return None + + def timeout(self) -> None: + if not self._cancelled: + for task in set(self._tasks): + task.cancel() + + self._cancelled = True + + +def ceil_timeout( + delay: Optional[float], ceil_threshold: float = 5 +) -> async_timeout.Timeout: + if delay is None or delay <= 0: + return async_timeout.timeout(None) + + loop = asyncio.get_running_loop() + now = loop.time() + when = now + delay + if delay > ceil_threshold: + when = ceil(when) + return async_timeout.timeout_at(when) + + +class HeadersMixin: + """Mixin for handling headers.""" + + ATTRS = frozenset(["_content_type", "_content_dict", "_stored_content_type"]) + + _headers: MultiMapping[str] + _content_type: Optional[str] = None + _content_dict: Optional[Dict[str, str]] = None + _stored_content_type: Union[str, None, _SENTINEL] = sentinel + + def _parse_content_type(self, raw: Optional[str]) -> None: + self._stored_content_type = raw + if raw is None: + # default value according to RFC 2616 + self._content_type = "application/octet-stream" + self._content_dict = {} + else: + content_type, content_mapping_proxy = parse_content_type(raw) + self._content_type = content_type + # _content_dict needs to be mutable so we can update it + self._content_dict = content_mapping_proxy.copy() + + @property + def content_type(self) -> str: + """The value of content part for Content-Type HTTP header.""" + raw = self._headers.get(hdrs.CONTENT_TYPE) + if self._stored_content_type != raw: + self._parse_content_type(raw) + assert self._content_type is not None + return self._content_type + + @property + def charset(self) -> Optional[str]: + """The value of charset part for Content-Type HTTP header.""" + raw = self._headers.get(hdrs.CONTENT_TYPE) + if self._stored_content_type != raw: + self._parse_content_type(raw) + assert self._content_dict is not None + return self._content_dict.get("charset") + + @property + def content_length(self) -> Optional[int]: + """The value of Content-Length HTTP header.""" + content_length = self._headers.get(hdrs.CONTENT_LENGTH) + return None if content_length is None else int(content_length) + + +def set_result(fut: "asyncio.Future[_T]", result: _T) -> None: + if not fut.done(): + fut.set_result(result) + + +_EXC_SENTINEL = BaseException() + + +class ErrorableProtocol(Protocol): + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = ..., + ) -> None: ... # pragma: no cover + + +def set_exception( + fut: "asyncio.Future[_T] | ErrorableProtocol", + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, +) -> None: + """Set future exception. + + If the future is marked as complete, this function is a no-op. + + :param exc_cause: An exception that is a direct cause of ``exc``. + Only set if provided. + """ + if asyncio.isfuture(fut) and fut.done(): + return + + exc_is_sentinel = exc_cause is _EXC_SENTINEL + exc_causes_itself = exc is exc_cause + if not exc_is_sentinel and not exc_causes_itself: + exc.__cause__ = exc_cause + + fut.set_exception(exc) + + +@functools.total_ordering +class AppKey(Generic[_T]): + """Keys for static typing support in Application.""" + + __slots__ = ("_name", "_t", "__orig_class__") + + # This may be set by Python when instantiating with a generic type. We need to + # support this, in order to support types that are not concrete classes, + # like Iterable, which can't be passed as the second parameter to __init__. + __orig_class__: Type[object] + + def __init__(self, name: str, t: Optional[Type[_T]] = None): + # Prefix with module name to help deduplicate key names. + frame = inspect.currentframe() + while frame: + if frame.f_code.co_name == "": + module: str = frame.f_globals["__name__"] + break + frame = frame.f_back + + self._name = module + "." + name + self._t = t + + def __lt__(self, other: object) -> bool: + if isinstance(other, AppKey): + return self._name < other._name + return True # Order AppKey above other types. + + def __repr__(self) -> str: + t = self._t + if t is None: + with suppress(AttributeError): + # Set to type arg. + t = get_args(self.__orig_class__)[0] + + if t is None: + t_repr = "<>" + elif isinstance(t, type): + if t.__module__ == "builtins": + t_repr = t.__qualname__ + else: + t_repr = f"{t.__module__}.{t.__qualname__}" + else: + t_repr = repr(t) + return f"" + + +class ChainMapProxy(Mapping[Union[str, AppKey[Any]], Any]): + __slots__ = ("_maps",) + + def __init__(self, maps: Iterable[Mapping[Union[str, AppKey[Any]], Any]]) -> None: + self._maps = tuple(maps) + + def __init_subclass__(cls) -> None: + raise TypeError( + "Inheritance class {} from ChainMapProxy " + "is forbidden".format(cls.__name__) + ) + + @overload # type: ignore[override] + def __getitem__(self, key: AppKey[_T]) -> _T: ... + + @overload + def __getitem__(self, key: str) -> Any: ... + + def __getitem__(self, key: Union[str, AppKey[_T]]) -> Any: + for mapping in self._maps: + try: + return mapping[key] + except KeyError: + pass + raise KeyError(key) + + @overload # type: ignore[override] + def get(self, key: AppKey[_T], default: _S) -> Union[_T, _S]: ... + + @overload + def get(self, key: AppKey[_T], default: None = ...) -> Optional[_T]: ... + + @overload + def get(self, key: str, default: Any = ...) -> Any: ... + + def get(self, key: Union[str, AppKey[_T]], default: Any = None) -> Any: + try: + return self[key] + except KeyError: + return default + + def __len__(self) -> int: + # reuses stored hash values if possible + return len(set().union(*self._maps)) + + def __iter__(self) -> Iterator[Union[str, AppKey[Any]]]: + d: Dict[Union[str, AppKey[Any]], Any] = {} + for mapping in reversed(self._maps): + # reuses stored hash values if possible + d.update(mapping) + return iter(d) + + def __contains__(self, key: object) -> bool: + return any(key in m for m in self._maps) + + def __bool__(self) -> bool: + return any(self._maps) + + def __repr__(self) -> str: + content = ", ".join(map(repr, self._maps)) + return f"ChainMapProxy({content})" + + +# https://tools.ietf.org/html/rfc7232#section-2.3 +_ETAGC = r"[!\x23-\x7E\x80-\xff]+" +_ETAGC_RE = re.compile(_ETAGC) +_QUOTED_ETAG = rf'(W/)?"({_ETAGC})"' +QUOTED_ETAG_RE = re.compile(_QUOTED_ETAG) +LIST_QUOTED_ETAG_RE = re.compile(rf"({_QUOTED_ETAG})(?:\s*,\s*|$)|(.)") + +ETAG_ANY = "*" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ETag: + value: str + is_weak: bool = False + + +def validate_etag_value(value: str) -> None: + if value != ETAG_ANY and not _ETAGC_RE.fullmatch(value): + raise ValueError( + f"Value {value!r} is not a valid etag. Maybe it contains '\"'?" + ) + + +def parse_http_date(date_str: Optional[str]) -> Optional[datetime.datetime]: + """Process a date string, return a datetime object""" + if date_str is not None: + timetuple = parsedate(date_str) + if timetuple is not None: + with suppress(ValueError): + return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc) + return None + + +@functools.lru_cache +def must_be_empty_body(method: str, code: int) -> bool: + """Check if a request must return an empty body.""" + return ( + code in EMPTY_BODY_STATUS_CODES + or method in EMPTY_BODY_METHODS + or (200 <= code < 300 and method in hdrs.METH_CONNECT_ALL) + ) + + +def should_remove_content_length(method: str, code: int) -> bool: + """Check if a Content-Length header should be removed. + + This should always be a subset of must_be_empty_body + """ + # https://www.rfc-editor.org/rfc/rfc9110.html#section-8.6-8 + # https://www.rfc-editor.org/rfc/rfc9110.html#section-15.4.5-4 + return code in EMPTY_BODY_STATUS_CODES or ( + 200 <= code < 300 and method in hdrs.METH_CONNECT_ALL + ) diff --git a/lib/python3.1/site-packages/aiohttp/http.py b/lib/python3.1/site-packages/aiohttp/http.py new file mode 100644 index 0000000000000000000000000000000000000000..a1feae2d9b8fe631d539a15dbf8e5ea2914d70d5 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/http.py @@ -0,0 +1,72 @@ +import sys +from http import HTTPStatus +from typing import Mapping, Tuple + +from . import __version__ +from .http_exceptions import HttpProcessingError as HttpProcessingError +from .http_parser import ( + HeadersParser as HeadersParser, + HttpParser as HttpParser, + HttpRequestParser as HttpRequestParser, + HttpResponseParser as HttpResponseParser, + RawRequestMessage as RawRequestMessage, + RawResponseMessage as RawResponseMessage, +) +from .http_websocket import ( + WS_CLOSED_MESSAGE as WS_CLOSED_MESSAGE, + WS_CLOSING_MESSAGE as WS_CLOSING_MESSAGE, + WS_KEY as WS_KEY, + WebSocketError as WebSocketError, + WebSocketReader as WebSocketReader, + WebSocketWriter as WebSocketWriter, + WSCloseCode as WSCloseCode, + WSMessage as WSMessage, + WSMsgType as WSMsgType, + ws_ext_gen as ws_ext_gen, + ws_ext_parse as ws_ext_parse, +) +from .http_writer import ( + HttpVersion as HttpVersion, + HttpVersion10 as HttpVersion10, + HttpVersion11 as HttpVersion11, + StreamWriter as StreamWriter, +) + +__all__ = ( + "HttpProcessingError", + "RESPONSES", + "SERVER_SOFTWARE", + # .http_writer + "StreamWriter", + "HttpVersion", + "HttpVersion10", + "HttpVersion11", + # .http_parser + "HeadersParser", + "HttpParser", + "HttpRequestParser", + "HttpResponseParser", + "RawRequestMessage", + "RawResponseMessage", + # .http_websocket + "WS_CLOSED_MESSAGE", + "WS_CLOSING_MESSAGE", + "WS_KEY", + "WebSocketReader", + "WebSocketWriter", + "ws_ext_gen", + "ws_ext_parse", + "WSMessage", + "WebSocketError", + "WSMsgType", + "WSCloseCode", +) + + +SERVER_SOFTWARE: str = "Python/{0[0]}.{0[1]} aiohttp/{1}".format( + sys.version_info, __version__ +) + +RESPONSES: Mapping[int, Tuple[str, str]] = { + v: (v.phrase, v.description) for v in HTTPStatus.__members__.values() +} diff --git a/lib/python3.1/site-packages/aiohttp/http_exceptions.py b/lib/python3.1/site-packages/aiohttp/http_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..773830211e6e654147afb000992782fb8bff4db8 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/http_exceptions.py @@ -0,0 +1,112 @@ +"""Low-level http related exceptions.""" + +from textwrap import indent +from typing import Optional, Union + +from .typedefs import _CIMultiDict + +__all__ = ("HttpProcessingError",) + + +class HttpProcessingError(Exception): + """HTTP error. + + Shortcut for raising HTTP errors with custom code, message and headers. + + code: HTTP Error code. + message: (optional) Error message. + headers: (optional) Headers to be sent in response, a list of pairs + """ + + code = 0 + message = "" + headers = None + + def __init__( + self, + *, + code: Optional[int] = None, + message: str = "", + headers: Optional[_CIMultiDict] = None, + ) -> None: + if code is not None: + self.code = code + self.headers = headers + self.message = message + + def __str__(self) -> str: + msg = indent(self.message, " ") + return f"{self.code}, message:\n{msg}" + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}: {self.code}, message={self.message!r}>" + + +class BadHttpMessage(HttpProcessingError): + + code = 400 + message = "Bad Request" + + def __init__(self, message: str, *, headers: Optional[_CIMultiDict] = None) -> None: + super().__init__(message=message, headers=headers) + self.args = (message,) + + +class HttpBadRequest(BadHttpMessage): + + code = 400 + message = "Bad Request" + + +class PayloadEncodingError(BadHttpMessage): + """Base class for payload errors""" + + +class ContentEncodingError(PayloadEncodingError): + """Content encoding error.""" + + +class TransferEncodingError(PayloadEncodingError): + """transfer encoding error.""" + + +class ContentLengthError(PayloadEncodingError): + """Not enough data to satisfy content length header.""" + + +class LineTooLong(BadHttpMessage): + def __init__( + self, line: str, limit: str = "Unknown", actual_size: str = "Unknown" + ) -> None: + super().__init__( + f"Got more than {limit} bytes ({actual_size}) when reading {line}." + ) + self.args = (line, limit, actual_size) + + +class InvalidHeader(BadHttpMessage): + def __init__(self, hdr: Union[bytes, str]) -> None: + hdr_s = hdr.decode(errors="backslashreplace") if isinstance(hdr, bytes) else hdr + super().__init__(f"Invalid HTTP header: {hdr!r}") + self.hdr = hdr_s + self.args = (hdr,) + + +class BadStatusLine(BadHttpMessage): + def __init__(self, line: str = "", error: Optional[str] = None) -> None: + if not isinstance(line, str): + line = repr(line) + super().__init__(error or f"Bad status line {line!r}") + self.args = (line,) + self.line = line + + +class BadHttpMethod(BadStatusLine): + """Invalid HTTP method in status line.""" + + def __init__(self, line: str = "", error: Optional[str] = None) -> None: + super().__init__(line, error or f"Bad HTTP method in status line {line!r}") + + +class InvalidURLError(BadHttpMessage): + pass diff --git a/lib/python3.1/site-packages/aiohttp/http_parser.py b/lib/python3.1/site-packages/aiohttp/http_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..9f864b2787666060bb4a5b98717abaf019d843fd --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/http_parser.py @@ -0,0 +1,1050 @@ +import abc +import asyncio +import re +import string +from contextlib import suppress +from enum import IntEnum +from typing import ( + Any, + ClassVar, + Final, + Generic, + List, + Literal, + NamedTuple, + Optional, + Pattern, + Set, + Tuple, + Type, + TypeVar, + Union, +) + +from multidict import CIMultiDict, CIMultiDictProxy, istr +from yarl import URL + +from . import hdrs +from .base_protocol import BaseProtocol +from .compression_utils import HAS_BROTLI, BrotliDecompressor, ZLibDecompressor +from .helpers import ( + _EXC_SENTINEL, + DEBUG, + EMPTY_BODY_METHODS, + EMPTY_BODY_STATUS_CODES, + NO_EXTENSIONS, + BaseTimerContext, + set_exception, +) +from .http_exceptions import ( + BadHttpMessage, + BadHttpMethod, + BadStatusLine, + ContentEncodingError, + ContentLengthError, + InvalidHeader, + InvalidURLError, + LineTooLong, + TransferEncodingError, +) +from .http_writer import HttpVersion, HttpVersion10 +from .streams import EMPTY_PAYLOAD, StreamReader +from .typedefs import RawHeaders + +__all__ = ( + "HeadersParser", + "HttpParser", + "HttpRequestParser", + "HttpResponseParser", + "RawRequestMessage", + "RawResponseMessage", +) + +_SEP = Literal[b"\r\n", b"\n"] + +ASCIISET: Final[Set[str]] = set(string.printable) + +# See https://www.rfc-editor.org/rfc/rfc9110.html#name-overview +# and https://www.rfc-editor.org/rfc/rfc9110.html#name-tokens +# +# method = token +# tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." / +# "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA +# token = 1*tchar +_TCHAR_SPECIALS: Final[str] = re.escape("!#$%&'*+-.^_`|~") +TOKENRE: Final[Pattern[str]] = re.compile(f"[0-9A-Za-z{_TCHAR_SPECIALS}]+") +VERSRE: Final[Pattern[str]] = re.compile(r"HTTP/(\d)\.(\d)", re.ASCII) +DIGITS: Final[Pattern[str]] = re.compile(r"\d+", re.ASCII) +HEXDIGITS: Final[Pattern[bytes]] = re.compile(rb"[0-9a-fA-F]+") + + +class RawRequestMessage(NamedTuple): + method: str + path: str + version: HttpVersion + headers: "CIMultiDictProxy[str]" + raw_headers: RawHeaders + should_close: bool + compression: Optional[str] + upgrade: bool + chunked: bool + url: URL + + +class RawResponseMessage(NamedTuple): + version: HttpVersion + code: int + reason: str + headers: CIMultiDictProxy[str] + raw_headers: RawHeaders + should_close: bool + compression: Optional[str] + upgrade: bool + chunked: bool + + +_MsgT = TypeVar("_MsgT", RawRequestMessage, RawResponseMessage) + + +class ParseState(IntEnum): + + PARSE_NONE = 0 + PARSE_LENGTH = 1 + PARSE_CHUNKED = 2 + PARSE_UNTIL_EOF = 3 + + +class ChunkState(IntEnum): + PARSE_CHUNKED_SIZE = 0 + PARSE_CHUNKED_CHUNK = 1 + PARSE_CHUNKED_CHUNK_EOF = 2 + PARSE_MAYBE_TRAILERS = 3 + PARSE_TRAILERS = 4 + + +class HeadersParser: + def __init__( + self, + max_line_size: int = 8190, + max_headers: int = 32768, + max_field_size: int = 8190, + lax: bool = False, + ) -> None: + self.max_line_size = max_line_size + self.max_headers = max_headers + self.max_field_size = max_field_size + self._lax = lax + + def parse_headers( + self, lines: List[bytes] + ) -> Tuple["CIMultiDictProxy[str]", RawHeaders]: + headers: CIMultiDict[str] = CIMultiDict() + # note: "raw" does not mean inclusion of OWS before/after the field value + raw_headers = [] + + lines_idx = 0 + line = lines[lines_idx] + line_count = len(lines) + + while line: + # Parse initial header name : value pair. + try: + bname, bvalue = line.split(b":", 1) + except ValueError: + raise InvalidHeader(line) from None + + if len(bname) == 0: + raise InvalidHeader(bname) + + # https://www.rfc-editor.org/rfc/rfc9112.html#section-5.1-2 + if {bname[0], bname[-1]} & {32, 9}: # {" ", "\t"} + raise InvalidHeader(line) + + bvalue = bvalue.lstrip(b" \t") + if len(bname) > self.max_field_size: + raise LineTooLong( + "request header name {}".format( + bname.decode("utf8", "backslashreplace") + ), + str(self.max_field_size), + str(len(bname)), + ) + name = bname.decode("utf-8", "surrogateescape") + if not TOKENRE.fullmatch(name): + raise InvalidHeader(bname) + + header_length = len(bvalue) + + # next line + lines_idx += 1 + line = lines[lines_idx] + + # consume continuation lines + continuation = self._lax and line and line[0] in (32, 9) # (' ', '\t') + + # Deprecated: https://www.rfc-editor.org/rfc/rfc9112.html#name-obsolete-line-folding + if continuation: + bvalue_lst = [bvalue] + while continuation: + header_length += len(line) + if header_length > self.max_field_size: + raise LineTooLong( + "request header field {}".format( + bname.decode("utf8", "backslashreplace") + ), + str(self.max_field_size), + str(header_length), + ) + bvalue_lst.append(line) + + # next line + lines_idx += 1 + if lines_idx < line_count: + line = lines[lines_idx] + if line: + continuation = line[0] in (32, 9) # (' ', '\t') + else: + line = b"" + break + bvalue = b"".join(bvalue_lst) + else: + if header_length > self.max_field_size: + raise LineTooLong( + "request header field {}".format( + bname.decode("utf8", "backslashreplace") + ), + str(self.max_field_size), + str(header_length), + ) + + bvalue = bvalue.strip(b" \t") + value = bvalue.decode("utf-8", "surrogateescape") + + # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.5-5 + if "\n" in value or "\r" in value or "\x00" in value: + raise InvalidHeader(bvalue) + + headers.add(name, value) + raw_headers.append((bname, bvalue)) + + return (CIMultiDictProxy(headers), tuple(raw_headers)) + + +def _is_supported_upgrade(headers: CIMultiDictProxy[str]) -> bool: + """Check if the upgrade header is supported.""" + return headers.get(hdrs.UPGRADE, "").lower() in {"tcp", "websocket"} + + +class HttpParser(abc.ABC, Generic[_MsgT]): + lax: ClassVar[bool] = False + + def __init__( + self, + protocol: Optional[BaseProtocol] = None, + loop: Optional[asyncio.AbstractEventLoop] = None, + limit: int = 2**16, + max_line_size: int = 8190, + max_headers: int = 32768, + max_field_size: int = 8190, + timer: Optional[BaseTimerContext] = None, + code: Optional[int] = None, + method: Optional[str] = None, + payload_exception: Optional[Type[BaseException]] = None, + response_with_body: bool = True, + read_until_eof: bool = False, + auto_decompress: bool = True, + ) -> None: + self.protocol = protocol + self.loop = loop + self.max_line_size = max_line_size + self.max_headers = max_headers + self.max_field_size = max_field_size + self.timer = timer + self.code = code + self.method = method + self.payload_exception = payload_exception + self.response_with_body = response_with_body + self.read_until_eof = read_until_eof + + self._lines: List[bytes] = [] + self._tail = b"" + self._upgraded = False + self._payload = None + self._payload_parser: Optional[HttpPayloadParser] = None + self._auto_decompress = auto_decompress + self._limit = limit + self._headers_parser = HeadersParser( + max_line_size, max_headers, max_field_size, self.lax + ) + + @abc.abstractmethod + def parse_message(self, lines: List[bytes]) -> _MsgT: ... + + @abc.abstractmethod + def _is_chunked_te(self, te: str) -> bool: ... + + def feed_eof(self) -> Optional[_MsgT]: + if self._payload_parser is not None: + self._payload_parser.feed_eof() + self._payload_parser = None + else: + # try to extract partial message + if self._tail: + self._lines.append(self._tail) + + if self._lines: + if self._lines[-1] != "\r\n": + self._lines.append(b"") + with suppress(Exception): + return self.parse_message(self._lines) + return None + + def feed_data( + self, + data: bytes, + SEP: _SEP = b"\r\n", + EMPTY: bytes = b"", + CONTENT_LENGTH: istr = hdrs.CONTENT_LENGTH, + METH_CONNECT: str = hdrs.METH_CONNECT, + SEC_WEBSOCKET_KEY1: istr = hdrs.SEC_WEBSOCKET_KEY1, + ) -> Tuple[List[Tuple[_MsgT, StreamReader]], bool, bytes]: + + messages = [] + + if self._tail: + data, self._tail = self._tail + data, b"" + + data_len = len(data) + start_pos = 0 + loop = self.loop + + should_close = False + while start_pos < data_len: + + # read HTTP message (request/response line + headers), \r\n\r\n + # and split by lines + if self._payload_parser is None and not self._upgraded: + pos = data.find(SEP, start_pos) + # consume \r\n + if pos == start_pos and not self._lines: + start_pos = pos + len(SEP) + continue + + if pos >= start_pos: + if should_close: + raise BadHttpMessage("Data after `Connection: close`") + + # line found + line = data[start_pos:pos] + if SEP == b"\n": # For lax response parsing + line = line.rstrip(b"\r") + self._lines.append(line) + start_pos = pos + len(SEP) + + # \r\n\r\n found + if self._lines[-1] == EMPTY: + try: + msg: _MsgT = self.parse_message(self._lines) + finally: + self._lines.clear() + + def get_content_length() -> Optional[int]: + # payload length + length_hdr = msg.headers.get(CONTENT_LENGTH) + if length_hdr is None: + return None + + # Shouldn't allow +/- or other number formats. + # https://www.rfc-editor.org/rfc/rfc9110#section-8.6-2 + # msg.headers is already stripped of leading/trailing wsp + if not DIGITS.fullmatch(length_hdr): + raise InvalidHeader(CONTENT_LENGTH) + + return int(length_hdr) + + length = get_content_length() + # do not support old websocket spec + if SEC_WEBSOCKET_KEY1 in msg.headers: + raise InvalidHeader(SEC_WEBSOCKET_KEY1) + + self._upgraded = msg.upgrade and _is_supported_upgrade( + msg.headers + ) + + method = getattr(msg, "method", self.method) + # code is only present on responses + code = getattr(msg, "code", 0) + + assert self.protocol is not None + # calculate payload + empty_body = code in EMPTY_BODY_STATUS_CODES or bool( + method and method in EMPTY_BODY_METHODS + ) + if not empty_body and ( + ((length is not None and length > 0) or msg.chunked) + and not self._upgraded + ): + payload = StreamReader( + self.protocol, + timer=self.timer, + loop=loop, + limit=self._limit, + ) + payload_parser = HttpPayloadParser( + payload, + length=length, + chunked=msg.chunked, + method=method, + compression=msg.compression, + code=self.code, + response_with_body=self.response_with_body, + auto_decompress=self._auto_decompress, + lax=self.lax, + headers_parser=self._headers_parser, + ) + if not payload_parser.done: + self._payload_parser = payload_parser + elif method == METH_CONNECT: + assert isinstance(msg, RawRequestMessage) + payload = StreamReader( + self.protocol, + timer=self.timer, + loop=loop, + limit=self._limit, + ) + self._upgraded = True + self._payload_parser = HttpPayloadParser( + payload, + method=msg.method, + compression=msg.compression, + auto_decompress=self._auto_decompress, + lax=self.lax, + headers_parser=self._headers_parser, + ) + elif not empty_body and length is None and self.read_until_eof: + payload = StreamReader( + self.protocol, + timer=self.timer, + loop=loop, + limit=self._limit, + ) + payload_parser = HttpPayloadParser( + payload, + length=length, + chunked=msg.chunked, + method=method, + compression=msg.compression, + code=self.code, + response_with_body=self.response_with_body, + auto_decompress=self._auto_decompress, + lax=self.lax, + headers_parser=self._headers_parser, + ) + if not payload_parser.done: + self._payload_parser = payload_parser + else: + payload = EMPTY_PAYLOAD + + messages.append((msg, payload)) + should_close = msg.should_close + else: + self._tail = data[start_pos:] + data = EMPTY + break + + # no parser, just store + elif self._payload_parser is None and self._upgraded: + assert not self._lines + break + + # feed payload + elif data and start_pos < data_len: + assert not self._lines + assert self._payload_parser is not None + try: + eof, data = self._payload_parser.feed_data(data[start_pos:], SEP) + except BaseException as underlying_exc: + reraised_exc = underlying_exc + if self.payload_exception is not None: + reraised_exc = self.payload_exception(str(underlying_exc)) + + set_exception( + self._payload_parser.payload, + reraised_exc, + underlying_exc, + ) + + eof = True + data = b"" + if isinstance( + underlying_exc, (InvalidHeader, TransferEncodingError) + ): + raise + + if eof: + start_pos = 0 + data_len = len(data) + self._payload_parser = None + continue + else: + break + + if data and start_pos < data_len: + data = data[start_pos:] + else: + data = EMPTY + + return messages, self._upgraded, data + + def parse_headers( + self, lines: List[bytes] + ) -> Tuple[ + "CIMultiDictProxy[str]", RawHeaders, Optional[bool], Optional[str], bool, bool + ]: + """Parses RFC 5322 headers from a stream. + + Line continuations are supported. Returns list of header name + and value pairs. Header name is in upper case. + """ + headers, raw_headers = self._headers_parser.parse_headers(lines) + close_conn = None + encoding = None + upgrade = False + chunked = False + + # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.5-6 + # https://www.rfc-editor.org/rfc/rfc9110.html#name-collected-abnf + singletons = ( + hdrs.CONTENT_LENGTH, + hdrs.CONTENT_LOCATION, + hdrs.CONTENT_RANGE, + hdrs.CONTENT_TYPE, + hdrs.ETAG, + hdrs.HOST, + hdrs.MAX_FORWARDS, + hdrs.SERVER, + hdrs.TRANSFER_ENCODING, + hdrs.USER_AGENT, + ) + bad_hdr = next((h for h in singletons if len(headers.getall(h, ())) > 1), None) + if bad_hdr is not None: + raise BadHttpMessage(f"Duplicate '{bad_hdr}' header found.") + + # keep-alive + conn = headers.get(hdrs.CONNECTION) + if conn: + v = conn.lower() + if v == "close": + close_conn = True + elif v == "keep-alive": + close_conn = False + # https://www.rfc-editor.org/rfc/rfc9110.html#name-101-switching-protocols + elif v == "upgrade" and headers.get(hdrs.UPGRADE): + upgrade = True + + # encoding + enc = headers.get(hdrs.CONTENT_ENCODING) + if enc: + enc = enc.lower() + if enc in ("gzip", "deflate", "br"): + encoding = enc + + # chunking + te = headers.get(hdrs.TRANSFER_ENCODING) + if te is not None: + if self._is_chunked_te(te): + chunked = True + + if hdrs.CONTENT_LENGTH in headers: + raise BadHttpMessage( + "Transfer-Encoding can't be present with Content-Length", + ) + + return (headers, raw_headers, close_conn, encoding, upgrade, chunked) + + def set_upgraded(self, val: bool) -> None: + """Set connection upgraded (to websocket) mode. + + :param bool val: new state. + """ + self._upgraded = val + + +class HttpRequestParser(HttpParser[RawRequestMessage]): + """Read request status line. + + Exception .http_exceptions.BadStatusLine + could be raised in case of any errors in status line. + Returns RawRequestMessage. + """ + + def parse_message(self, lines: List[bytes]) -> RawRequestMessage: + # request line + line = lines[0].decode("utf-8", "surrogateescape") + try: + method, path, version = line.split(" ", maxsplit=2) + except ValueError: + raise BadHttpMethod(line) from None + + if len(path) > self.max_line_size: + raise LineTooLong( + "Status line is too long", str(self.max_line_size), str(len(path)) + ) + + # method + if not TOKENRE.fullmatch(method): + raise BadHttpMethod(method) + + # version + match = VERSRE.fullmatch(version) + if match is None: + raise BadStatusLine(line) + version_o = HttpVersion(int(match.group(1)), int(match.group(2))) + + if method == "CONNECT": + # authority-form, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3 + url = URL.build(authority=path, encoded=True) + elif path.startswith("/"): + # origin-form, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.1 + path_part, _hash_separator, url_fragment = path.partition("#") + path_part, _question_mark_separator, qs_part = path_part.partition("?") + + # NOTE: `yarl.URL.build()` is used to mimic what the Cython-based + # NOTE: parser does, otherwise it results into the same + # NOTE: HTTP Request-Line input producing different + # NOTE: `yarl.URL()` objects + url = URL.build( + path=path_part, + query_string=qs_part, + fragment=url_fragment, + encoded=True, + ) + elif path == "*" and method == "OPTIONS": + # asterisk-form, + url = URL(path, encoded=True) + else: + # absolute-form for proxy maybe, + # https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2 + url = URL(path, encoded=True) + if url.scheme == "": + # not absolute-form + raise InvalidURLError( + path.encode(errors="surrogateescape").decode("latin1") + ) + + # read headers + ( + headers, + raw_headers, + close, + compression, + upgrade, + chunked, + ) = self.parse_headers(lines[1:]) + + if close is None: # then the headers weren't set in the request + if version_o <= HttpVersion10: # HTTP 1.0 must asks to not close + close = True + else: # HTTP 1.1 must ask to close. + close = False + + return RawRequestMessage( + method, + path, + version_o, + headers, + raw_headers, + close, + compression, + upgrade, + chunked, + url, + ) + + def _is_chunked_te(self, te: str) -> bool: + if te.rsplit(",", maxsplit=1)[-1].strip(" \t").lower() == "chunked": + return True + # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.3 + raise BadHttpMessage("Request has invalid `Transfer-Encoding`") + + +class HttpResponseParser(HttpParser[RawResponseMessage]): + """Read response status line and headers. + + BadStatusLine could be raised in case of any errors in status line. + Returns RawResponseMessage. + """ + + # Lax mode should only be enabled on response parser. + lax = not DEBUG + + def feed_data( + self, + data: bytes, + SEP: Optional[_SEP] = None, + *args: Any, + **kwargs: Any, + ) -> Tuple[List[Tuple[RawResponseMessage, StreamReader]], bool, bytes]: + if SEP is None: + SEP = b"\r\n" if DEBUG else b"\n" + return super().feed_data(data, SEP, *args, **kwargs) + + def parse_message(self, lines: List[bytes]) -> RawResponseMessage: + line = lines[0].decode("utf-8", "surrogateescape") + try: + version, status = line.split(maxsplit=1) + except ValueError: + raise BadStatusLine(line) from None + + try: + status, reason = status.split(maxsplit=1) + except ValueError: + status = status.strip() + reason = "" + + if len(reason) > self.max_line_size: + raise LineTooLong( + "Status line is too long", str(self.max_line_size), str(len(reason)) + ) + + # version + match = VERSRE.fullmatch(version) + if match is None: + raise BadStatusLine(line) + version_o = HttpVersion(int(match.group(1)), int(match.group(2))) + + # The status code is a three-digit ASCII number, no padding + if len(status) != 3 or not DIGITS.fullmatch(status): + raise BadStatusLine(line) + status_i = int(status) + + # read headers + ( + headers, + raw_headers, + close, + compression, + upgrade, + chunked, + ) = self.parse_headers(lines[1:]) + + if close is None: + if version_o <= HttpVersion10: + close = True + # https://www.rfc-editor.org/rfc/rfc9112.html#name-message-body-length + elif 100 <= status_i < 200 or status_i in {204, 304}: + close = False + elif hdrs.CONTENT_LENGTH in headers or hdrs.TRANSFER_ENCODING in headers: + close = False + else: + # https://www.rfc-editor.org/rfc/rfc9112.html#section-6.3-2.8 + close = True + + return RawResponseMessage( + version_o, + status_i, + reason.strip(), + headers, + raw_headers, + close, + compression, + upgrade, + chunked, + ) + + def _is_chunked_te(self, te: str) -> bool: + # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.2 + return te.rsplit(",", maxsplit=1)[-1].strip(" \t").lower() == "chunked" + + +class HttpPayloadParser: + def __init__( + self, + payload: StreamReader, + length: Optional[int] = None, + chunked: bool = False, + compression: Optional[str] = None, + code: Optional[int] = None, + method: Optional[str] = None, + response_with_body: bool = True, + auto_decompress: bool = True, + lax: bool = False, + *, + headers_parser: HeadersParser, + ) -> None: + self._length = 0 + self._type = ParseState.PARSE_UNTIL_EOF + self._chunk = ChunkState.PARSE_CHUNKED_SIZE + self._chunk_size = 0 + self._chunk_tail = b"" + self._auto_decompress = auto_decompress + self._lax = lax + self._headers_parser = headers_parser + self._trailer_lines: list[bytes] = [] + self.done = False + + # payload decompression wrapper + if response_with_body and compression and self._auto_decompress: + real_payload: Union[StreamReader, DeflateBuffer] = DeflateBuffer( + payload, compression + ) + else: + real_payload = payload + + # payload parser + if not response_with_body: + # don't parse payload if it's not expected to be received + self._type = ParseState.PARSE_NONE + real_payload.feed_eof() + self.done = True + elif chunked: + self._type = ParseState.PARSE_CHUNKED + elif length is not None: + self._type = ParseState.PARSE_LENGTH + self._length = length + if self._length == 0: + real_payload.feed_eof() + self.done = True + + self.payload = real_payload + + def feed_eof(self) -> None: + if self._type == ParseState.PARSE_UNTIL_EOF: + self.payload.feed_eof() + elif self._type == ParseState.PARSE_LENGTH: + raise ContentLengthError( + "Not enough data to satisfy content length header." + ) + elif self._type == ParseState.PARSE_CHUNKED: + raise TransferEncodingError( + "Not enough data to satisfy transfer length header." + ) + + def feed_data( + self, chunk: bytes, SEP: _SEP = b"\r\n", CHUNK_EXT: bytes = b";" + ) -> Tuple[bool, bytes]: + # Read specified amount of bytes + if self._type == ParseState.PARSE_LENGTH: + required = self._length + chunk_len = len(chunk) + + if required >= chunk_len: + self._length = required - chunk_len + self.payload.feed_data(chunk, chunk_len) + if self._length == 0: + self.payload.feed_eof() + return True, b"" + else: + self._length = 0 + self.payload.feed_data(chunk[:required], required) + self.payload.feed_eof() + return True, chunk[required:] + + # Chunked transfer encoding parser + elif self._type == ParseState.PARSE_CHUNKED: + if self._chunk_tail: + chunk = self._chunk_tail + chunk + self._chunk_tail = b"" + + while chunk: + + # read next chunk size + if self._chunk == ChunkState.PARSE_CHUNKED_SIZE: + pos = chunk.find(SEP) + if pos >= 0: + i = chunk.find(CHUNK_EXT, 0, pos) + if i >= 0: + size_b = chunk[:i] # strip chunk-extensions + # Verify no LF in the chunk-extension + if b"\n" in (ext := chunk[i:pos]): + exc = TransferEncodingError( + f"Unexpected LF in chunk-extension: {ext!r}" + ) + set_exception(self.payload, exc) + raise exc + else: + size_b = chunk[:pos] + + if self._lax: # Allow whitespace in lax mode. + size_b = size_b.strip() + + if not re.fullmatch(HEXDIGITS, size_b): + exc = TransferEncodingError( + chunk[:pos].decode("ascii", "surrogateescape") + ) + set_exception(self.payload, exc) + raise exc + size = int(bytes(size_b), 16) + + chunk = chunk[pos + len(SEP) :] + if size == 0: # eof marker + self._chunk = ChunkState.PARSE_TRAILERS + if self._lax and chunk.startswith(b"\r"): + chunk = chunk[1:] + else: + self._chunk = ChunkState.PARSE_CHUNKED_CHUNK + self._chunk_size = size + self.payload.begin_http_chunk_receiving() + else: + self._chunk_tail = chunk + return False, b"" + + # read chunk and feed buffer + if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK: + required = self._chunk_size + chunk_len = len(chunk) + + if required > chunk_len: + self._chunk_size = required - chunk_len + self.payload.feed_data(chunk, chunk_len) + return False, b"" + else: + self._chunk_size = 0 + self.payload.feed_data(chunk[:required], required) + chunk = chunk[required:] + self._chunk = ChunkState.PARSE_CHUNKED_CHUNK_EOF + self.payload.end_http_chunk_receiving() + + # toss the CRLF at the end of the chunk + if self._chunk == ChunkState.PARSE_CHUNKED_CHUNK_EOF: + if self._lax and chunk.startswith(b"\r"): + chunk = chunk[1:] + if chunk[: len(SEP)] == SEP: + chunk = chunk[len(SEP) :] + self._chunk = ChunkState.PARSE_CHUNKED_SIZE + else: + self._chunk_tail = chunk + return False, b"" + + if self._chunk == ChunkState.PARSE_TRAILERS: + pos = chunk.find(SEP) + if pos < 0: # No line found + self._chunk_tail = chunk + return False, b"" + + line = chunk[:pos] + chunk = chunk[pos + len(SEP) :] + if SEP == b"\n": # For lax response parsing + line = line.rstrip(b"\r") + self._trailer_lines.append(line) + + # \r\n\r\n found, end of stream + if self._trailer_lines[-1] == b"": + # Headers and trailers are defined the same way, + # so we reuse the HeadersParser here. + try: + trailers, raw_trailers = self._headers_parser.parse_headers( + self._trailer_lines + ) + finally: + self._trailer_lines.clear() + self.payload.feed_eof() + return True, chunk + + # Read all bytes until eof + elif self._type == ParseState.PARSE_UNTIL_EOF: + self.payload.feed_data(chunk, len(chunk)) + + return False, b"" + + +class DeflateBuffer: + """DeflateStream decompress stream and feed data into specified stream.""" + + decompressor: Any + + def __init__(self, out: StreamReader, encoding: Optional[str]) -> None: + self.out = out + self.size = 0 + self.encoding = encoding + self._started_decoding = False + + self.decompressor: Union[BrotliDecompressor, ZLibDecompressor] + if encoding == "br": + if not HAS_BROTLI: # pragma: no cover + raise ContentEncodingError( + "Can not decode content-encoding: brotli (br). " + "Please install `Brotli`" + ) + self.decompressor = BrotliDecompressor() + else: + self.decompressor = ZLibDecompressor(encoding=encoding) + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + set_exception(self.out, exc, exc_cause) + + def feed_data(self, chunk: bytes, size: int) -> None: + if not size: + return + + self.size += size + + # RFC1950 + # bits 0..3 = CM = 0b1000 = 8 = "deflate" + # bits 4..7 = CINFO = 1..7 = windows size. + if ( + not self._started_decoding + and self.encoding == "deflate" + and chunk[0] & 0xF != 8 + ): + # Change the decoder to decompress incorrectly compressed data + # Actually we should issue a warning about non-RFC-compliant data. + self.decompressor = ZLibDecompressor( + encoding=self.encoding, suppress_deflate_header=True + ) + + try: + chunk = self.decompressor.decompress_sync(chunk) + except Exception: + raise ContentEncodingError( + "Can not decode content-encoding: %s" % self.encoding + ) + + self._started_decoding = True + + if chunk: + self.out.feed_data(chunk, len(chunk)) + + def feed_eof(self) -> None: + chunk = self.decompressor.flush() + + if chunk or self.size > 0: + self.out.feed_data(chunk, len(chunk)) + if self.encoding == "deflate" and not self.decompressor.eof: + raise ContentEncodingError("deflate") + + self.out.feed_eof() + + def begin_http_chunk_receiving(self) -> None: + self.out.begin_http_chunk_receiving() + + def end_http_chunk_receiving(self) -> None: + self.out.end_http_chunk_receiving() + + +HttpRequestParserPy = HttpRequestParser +HttpResponseParserPy = HttpResponseParser +RawRequestMessagePy = RawRequestMessage +RawResponseMessagePy = RawResponseMessage + +try: + if not NO_EXTENSIONS: + from ._http_parser import ( # type: ignore[import-not-found,no-redef] + HttpRequestParser, + HttpResponseParser, + RawRequestMessage, + RawResponseMessage, + ) + + HttpRequestParserC = HttpRequestParser + HttpResponseParserC = HttpResponseParser + RawRequestMessageC = RawRequestMessage + RawResponseMessageC = RawResponseMessage +except ImportError: # pragma: no cover + pass diff --git a/lib/python3.1/site-packages/aiohttp/http_websocket.py b/lib/python3.1/site-packages/aiohttp/http_websocket.py new file mode 100644 index 0000000000000000000000000000000000000000..6b4b30e02b247e30e0c84d3eb118b749bbe52079 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/http_websocket.py @@ -0,0 +1,36 @@ +"""WebSocket protocol versions 13 and 8.""" + +from ._websocket.helpers import WS_KEY, ws_ext_gen, ws_ext_parse +from ._websocket.models import ( + WS_CLOSED_MESSAGE, + WS_CLOSING_MESSAGE, + WebSocketError, + WSCloseCode, + WSHandshakeError, + WSMessage, + WSMsgType, +) +from ._websocket.reader import WebSocketReader +from ._websocket.writer import WebSocketWriter + +# Messages that the WebSocketResponse.receive needs to handle internally +_INTERNAL_RECEIVE_TYPES = frozenset( + (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.PING, WSMsgType.PONG) +) + + +__all__ = ( + "WS_CLOSED_MESSAGE", + "WS_CLOSING_MESSAGE", + "WS_KEY", + "WebSocketReader", + "WebSocketWriter", + "WSMessage", + "WebSocketError", + "WSMsgType", + "WSCloseCode", + "ws_ext_gen", + "ws_ext_parse", + "WSHandshakeError", + "WSMessage", +) diff --git a/lib/python3.1/site-packages/aiohttp/http_writer.py b/lib/python3.1/site-packages/aiohttp/http_writer.py new file mode 100644 index 0000000000000000000000000000000000000000..a140b218b25fedccb49451c547d4d01326367cfb --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/http_writer.py @@ -0,0 +1,378 @@ +"""Http related parsers and protocol.""" + +import asyncio +import sys +from typing import ( # noqa + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Iterable, + List, + NamedTuple, + Optional, + Union, +) + +from multidict import CIMultiDict + +from .abc import AbstractStreamWriter +from .base_protocol import BaseProtocol +from .client_exceptions import ClientConnectionResetError +from .compression_utils import ZLibCompressor +from .helpers import NO_EXTENSIONS + +__all__ = ("StreamWriter", "HttpVersion", "HttpVersion10", "HttpVersion11") + + +MIN_PAYLOAD_FOR_WRITELINES = 2048 +IS_PY313_BEFORE_313_2 = (3, 13, 0) <= sys.version_info < (3, 13, 2) +IS_PY_BEFORE_312_9 = sys.version_info < (3, 12, 9) +SKIP_WRITELINES = IS_PY313_BEFORE_313_2 or IS_PY_BEFORE_312_9 +# writelines is not safe for use +# on Python 3.12+ until 3.12.9 +# on Python 3.13+ until 3.13.2 +# and on older versions it not any faster than write +# CVE-2024-12254: https://github.com/python/cpython/pull/127656 + + +class HttpVersion(NamedTuple): + major: int + minor: int + + +HttpVersion10 = HttpVersion(1, 0) +HttpVersion11 = HttpVersion(1, 1) + + +_T_OnChunkSent = Optional[Callable[[bytes], Awaitable[None]]] +_T_OnHeadersSent = Optional[Callable[["CIMultiDict[str]"], Awaitable[None]]] + + +class StreamWriter(AbstractStreamWriter): + + length: Optional[int] = None + chunked: bool = False + _eof: bool = False + _compress: Optional[ZLibCompressor] = None + + def __init__( + self, + protocol: BaseProtocol, + loop: asyncio.AbstractEventLoop, + on_chunk_sent: _T_OnChunkSent = None, + on_headers_sent: _T_OnHeadersSent = None, + ) -> None: + self._protocol = protocol + self.loop = loop + self._on_chunk_sent: _T_OnChunkSent = on_chunk_sent + self._on_headers_sent: _T_OnHeadersSent = on_headers_sent + self._headers_buf: Optional[bytes] = None + self._headers_written: bool = False + + @property + def transport(self) -> Optional[asyncio.Transport]: + return self._protocol.transport + + @property + def protocol(self) -> BaseProtocol: + return self._protocol + + def enable_chunking(self) -> None: + self.chunked = True + + def enable_compression( + self, encoding: str = "deflate", strategy: Optional[int] = None + ) -> None: + self._compress = ZLibCompressor(encoding=encoding, strategy=strategy) + + def _write(self, chunk: Union[bytes, bytearray, memoryview]) -> None: + size = len(chunk) + self.buffer_size += size + self.output_size += size + transport = self._protocol.transport + if transport is None or transport.is_closing(): + raise ClientConnectionResetError("Cannot write to closing transport") + transport.write(chunk) + + def _writelines(self, chunks: Iterable[bytes]) -> None: + size = 0 + for chunk in chunks: + size += len(chunk) + self.buffer_size += size + self.output_size += size + transport = self._protocol.transport + if transport is None or transport.is_closing(): + raise ClientConnectionResetError("Cannot write to closing transport") + if SKIP_WRITELINES or size < MIN_PAYLOAD_FOR_WRITELINES: + transport.write(b"".join(chunks)) + else: + transport.writelines(chunks) + + def _write_chunked_payload( + self, chunk: Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"] + ) -> None: + """Write a chunk with proper chunked encoding.""" + chunk_len_pre = f"{len(chunk):x}\r\n".encode("ascii") + self._writelines((chunk_len_pre, chunk, b"\r\n")) + + def _send_headers_with_payload( + self, + chunk: Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"], + is_eof: bool, + ) -> None: + """Send buffered headers with payload, coalescing into single write.""" + # Mark headers as written + self._headers_written = True + headers_buf = self._headers_buf + self._headers_buf = None + + if TYPE_CHECKING: + # Safe because callers (write() and write_eof()) only invoke this method + # after checking that self._headers_buf is truthy + assert headers_buf is not None + + if not self.chunked: + # Non-chunked: coalesce headers with body + if chunk: + self._writelines((headers_buf, chunk)) + else: + self._write(headers_buf) + return + + # Coalesce headers with chunked data + if chunk: + chunk_len_pre = f"{len(chunk):x}\r\n".encode("ascii") + if is_eof: + self._writelines((headers_buf, chunk_len_pre, chunk, b"\r\n0\r\n\r\n")) + else: + self._writelines((headers_buf, chunk_len_pre, chunk, b"\r\n")) + elif is_eof: + self._writelines((headers_buf, b"0\r\n\r\n")) + else: + self._write(headers_buf) + + async def write( + self, + chunk: Union[bytes, bytearray, memoryview], + *, + drain: bool = True, + LIMIT: int = 0x10000, + ) -> None: + """ + Writes chunk of data to a stream. + + write_eof() indicates end of stream. + writer can't be used after write_eof() method being called. + write() return drain future. + """ + if self._on_chunk_sent is not None: + await self._on_chunk_sent(chunk) + + if isinstance(chunk, memoryview): + if chunk.nbytes != len(chunk): + # just reshape it + chunk = chunk.cast("c") + + if self._compress is not None: + chunk = await self._compress.compress(chunk) + if not chunk: + return + + if self.length is not None: + chunk_len = len(chunk) + if self.length >= chunk_len: + self.length = self.length - chunk_len + else: + chunk = chunk[: self.length] + self.length = 0 + if not chunk: + return + + # Handle buffered headers for small payload optimization + if self._headers_buf and not self._headers_written: + self._send_headers_with_payload(chunk, False) + if drain and self.buffer_size > LIMIT: + self.buffer_size = 0 + await self.drain() + return + + if chunk: + if self.chunked: + self._write_chunked_payload(chunk) + else: + self._write(chunk) + + if drain and self.buffer_size > LIMIT: + self.buffer_size = 0 + await self.drain() + + async def write_headers( + self, status_line: str, headers: "CIMultiDict[str]" + ) -> None: + """Write headers to the stream.""" + if self._on_headers_sent is not None: + await self._on_headers_sent(headers) + # status + headers + buf = _serialize_headers(status_line, headers) + self._headers_written = False + self._headers_buf = buf + + def send_headers(self) -> None: + """Force sending buffered headers if not already sent.""" + if not self._headers_buf or self._headers_written: + return + + self._headers_written = True + headers_buf = self._headers_buf + self._headers_buf = None + + if TYPE_CHECKING: + # Safe because we only enter this block when self._headers_buf is truthy + assert headers_buf is not None + + self._write(headers_buf) + + def set_eof(self) -> None: + """Indicate that the message is complete.""" + if self._eof: + return + + # If headers haven't been sent yet, send them now + # This handles the case where there's no body at all + if self._headers_buf and not self._headers_written: + self._headers_written = True + headers_buf = self._headers_buf + self._headers_buf = None + + if TYPE_CHECKING: + # Safe because we only enter this block when self._headers_buf is truthy + assert headers_buf is not None + + # Combine headers and chunked EOF marker in a single write + if self.chunked: + self._writelines((headers_buf, b"0\r\n\r\n")) + else: + self._write(headers_buf) + elif self.chunked and self._headers_written: + # Headers already sent, just send the final chunk marker + self._write(b"0\r\n\r\n") + + self._eof = True + + async def write_eof(self, chunk: bytes = b"") -> None: + if self._eof: + return + + if chunk and self._on_chunk_sent is not None: + await self._on_chunk_sent(chunk) + + # Handle body/compression + if self._compress: + chunks: List[bytes] = [] + chunks_len = 0 + if chunk and (compressed_chunk := await self._compress.compress(chunk)): + chunks_len = len(compressed_chunk) + chunks.append(compressed_chunk) + + flush_chunk = self._compress.flush() + chunks_len += len(flush_chunk) + chunks.append(flush_chunk) + assert chunks_len + + # Send buffered headers with compressed data if not yet sent + if self._headers_buf and not self._headers_written: + self._headers_written = True + headers_buf = self._headers_buf + self._headers_buf = None + + if self.chunked: + # Coalesce headers with compressed chunked data + chunk_len_pre = f"{chunks_len:x}\r\n".encode("ascii") + self._writelines( + (headers_buf, chunk_len_pre, *chunks, b"\r\n0\r\n\r\n") + ) + else: + # Coalesce headers with compressed data + self._writelines((headers_buf, *chunks)) + await self.drain() + self._eof = True + return + + # Headers already sent, just write compressed data + if self.chunked: + chunk_len_pre = f"{chunks_len:x}\r\n".encode("ascii") + self._writelines((chunk_len_pre, *chunks, b"\r\n0\r\n\r\n")) + elif len(chunks) > 1: + self._writelines(chunks) + else: + self._write(chunks[0]) + await self.drain() + self._eof = True + return + + # No compression - send buffered headers if not yet sent + if self._headers_buf and not self._headers_written: + # Use helper to send headers with payload + self._send_headers_with_payload(chunk, True) + await self.drain() + self._eof = True + return + + # Handle remaining body + if self.chunked: + if chunk: + # Write final chunk with EOF marker + self._writelines( + (f"{len(chunk):x}\r\n".encode("ascii"), chunk, b"\r\n0\r\n\r\n") + ) + else: + self._write(b"0\r\n\r\n") + await self.drain() + self._eof = True + return + + if chunk: + self._write(chunk) + await self.drain() + + self._eof = True + + async def drain(self) -> None: + """Flush the write buffer. + + The intended use is to write + + await w.write(data) + await w.drain() + """ + protocol = self._protocol + if protocol.transport is not None and protocol._paused: + await protocol._drain_helper() + + +def _safe_header(string: str) -> str: + if "\r" in string or "\n" in string: + raise ValueError( + "Newline or carriage return detected in headers. " + "Potential header injection attack." + ) + return string + + +def _py_serialize_headers(status_line: str, headers: "CIMultiDict[str]") -> bytes: + headers_gen = (_safe_header(k) + ": " + _safe_header(v) for k, v in headers.items()) + line = status_line + "\r\n" + "\r\n".join(headers_gen) + "\r\n\r\n" + return line.encode("utf-8") + + +_serialize_headers = _py_serialize_headers + +try: + import aiohttp._http_writer as _http_writer # type: ignore[import-not-found] + + _c_serialize_headers = _http_writer._serialize_headers + if not NO_EXTENSIONS: + _serialize_headers = _c_serialize_headers +except ImportError: + pass diff --git a/lib/python3.1/site-packages/aiohttp/log.py b/lib/python3.1/site-packages/aiohttp/log.py new file mode 100644 index 0000000000000000000000000000000000000000..3cecea2bac185df741bccd0a32a5fef9cfe23299 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/log.py @@ -0,0 +1,8 @@ +import logging + +access_logger = logging.getLogger("aiohttp.access") +client_logger = logging.getLogger("aiohttp.client") +internal_logger = logging.getLogger("aiohttp.internal") +server_logger = logging.getLogger("aiohttp.server") +web_logger = logging.getLogger("aiohttp.web") +ws_logger = logging.getLogger("aiohttp.websocket") diff --git a/lib/python3.1/site-packages/aiohttp/multipart.py b/lib/python3.1/site-packages/aiohttp/multipart.py new file mode 100644 index 0000000000000000000000000000000000000000..54dfd4843b075c3764d20aa5d93e7ab9181c45d9 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/multipart.py @@ -0,0 +1,1141 @@ +import base64 +import binascii +import json +import re +import sys +import uuid +import warnings +from collections import deque +from collections.abc import Mapping, Sequence +from types import TracebackType +from typing import ( + TYPE_CHECKING, + Any, + Deque, + Dict, + Iterator, + List, + Optional, + Tuple, + Type, + Union, + cast, +) +from urllib.parse import parse_qsl, unquote, urlencode + +from multidict import CIMultiDict, CIMultiDictProxy + +from .compression_utils import ZLibCompressor, ZLibDecompressor +from .hdrs import ( + CONTENT_DISPOSITION, + CONTENT_ENCODING, + CONTENT_LENGTH, + CONTENT_TRANSFER_ENCODING, + CONTENT_TYPE, +) +from .helpers import CHAR, TOKEN, parse_mimetype, reify +from .http import HeadersParser +from .log import internal_logger +from .payload import ( + JsonPayload, + LookupError, + Order, + Payload, + StringPayload, + get_payload, + payload_type, +) +from .streams import StreamReader + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing import TypeVar + + Self = TypeVar("Self", bound="BodyPartReader") + +__all__ = ( + "MultipartReader", + "MultipartWriter", + "BodyPartReader", + "BadContentDispositionHeader", + "BadContentDispositionParam", + "parse_content_disposition", + "content_disposition_filename", +) + + +if TYPE_CHECKING: + from .client_reqrep import ClientResponse + + +class BadContentDispositionHeader(RuntimeWarning): + pass + + +class BadContentDispositionParam(RuntimeWarning): + pass + + +def parse_content_disposition( + header: Optional[str], +) -> Tuple[Optional[str], Dict[str, str]]: + def is_token(string: str) -> bool: + return bool(string) and TOKEN >= set(string) + + def is_quoted(string: str) -> bool: + return string[0] == string[-1] == '"' + + def is_rfc5987(string: str) -> bool: + return is_token(string) and string.count("'") == 2 + + def is_extended_param(string: str) -> bool: + return string.endswith("*") + + def is_continuous_param(string: str) -> bool: + pos = string.find("*") + 1 + if not pos: + return False + substring = string[pos:-1] if string.endswith("*") else string[pos:] + return substring.isdigit() + + def unescape(text: str, *, chars: str = "".join(map(re.escape, CHAR))) -> str: + return re.sub(f"\\\\([{chars}])", "\\1", text) + + if not header: + return None, {} + + disptype, *parts = header.split(";") + if not is_token(disptype): + warnings.warn(BadContentDispositionHeader(header)) + return None, {} + + params: Dict[str, str] = {} + while parts: + item = parts.pop(0) + + if "=" not in item: + warnings.warn(BadContentDispositionHeader(header)) + return None, {} + + key, value = item.split("=", 1) + key = key.lower().strip() + value = value.lstrip() + + if key in params: + warnings.warn(BadContentDispositionHeader(header)) + return None, {} + + if not is_token(key): + warnings.warn(BadContentDispositionParam(item)) + continue + + elif is_continuous_param(key): + if is_quoted(value): + value = unescape(value[1:-1]) + elif not is_token(value): + warnings.warn(BadContentDispositionParam(item)) + continue + + elif is_extended_param(key): + if is_rfc5987(value): + encoding, _, value = value.split("'", 2) + encoding = encoding or "utf-8" + else: + warnings.warn(BadContentDispositionParam(item)) + continue + + try: + value = unquote(value, encoding, "strict") + except UnicodeDecodeError: # pragma: nocover + warnings.warn(BadContentDispositionParam(item)) + continue + + else: + failed = True + if is_quoted(value): + failed = False + value = unescape(value[1:-1].lstrip("\\/")) + elif is_token(value): + failed = False + elif parts: + # maybe just ; in filename, in any case this is just + # one case fix, for proper fix we need to redesign parser + _value = f"{value};{parts[0]}" + if is_quoted(_value): + parts.pop(0) + value = unescape(_value[1:-1].lstrip("\\/")) + failed = False + + if failed: + warnings.warn(BadContentDispositionHeader(header)) + return None, {} + + params[key] = value + + return disptype.lower(), params + + +def content_disposition_filename( + params: Mapping[str, str], name: str = "filename" +) -> Optional[str]: + name_suf = "%s*" % name + if not params: + return None + elif name_suf in params: + return params[name_suf] + elif name in params: + return params[name] + else: + parts = [] + fnparams = sorted( + (key, value) for key, value in params.items() if key.startswith(name_suf) + ) + for num, (key, value) in enumerate(fnparams): + _, tail = key.split("*", 1) + if tail.endswith("*"): + tail = tail[:-1] + if tail == str(num): + parts.append(value) + else: + break + if not parts: + return None + value = "".join(parts) + if "'" in value: + encoding, _, value = value.split("'", 2) + encoding = encoding or "utf-8" + return unquote(value, encoding, "strict") + return value + + +class MultipartResponseWrapper: + """Wrapper around the MultipartReader. + + It takes care about + underlying connection and close it when it needs in. + """ + + def __init__( + self, + resp: "ClientResponse", + stream: "MultipartReader", + ) -> None: + self.resp = resp + self.stream = stream + + def __aiter__(self) -> "MultipartResponseWrapper": + return self + + async def __anext__( + self, + ) -> Union["MultipartReader", "BodyPartReader"]: + part = await self.next() + if part is None: + raise StopAsyncIteration + return part + + def at_eof(self) -> bool: + """Returns True when all response data had been read.""" + return self.resp.content.at_eof() + + async def next( + self, + ) -> Optional[Union["MultipartReader", "BodyPartReader"]]: + """Emits next multipart reader object.""" + item = await self.stream.next() + if self.stream.at_eof(): + await self.release() + return item + + async def release(self) -> None: + """Release the connection gracefully. + + All remaining content is read to the void. + """ + await self.resp.release() + + +class BodyPartReader: + """Multipart reader for single body part.""" + + chunk_size = 8192 + + def __init__( + self, + boundary: bytes, + headers: "CIMultiDictProxy[str]", + content: StreamReader, + *, + subtype: str = "mixed", + default_charset: Optional[str] = None, + ) -> None: + self.headers = headers + self._boundary = boundary + self._boundary_len = len(boundary) + 2 # Boundary + \r\n + self._content = content + self._default_charset = default_charset + self._at_eof = False + self._is_form_data = subtype == "form-data" + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.8 + length = None if self._is_form_data else self.headers.get(CONTENT_LENGTH, None) + self._length = int(length) if length is not None else None + self._read_bytes = 0 + self._unread: Deque[bytes] = deque() + self._prev_chunk: Optional[bytes] = None + self._content_eof = 0 + self._cache: Dict[str, Any] = {} + + def __aiter__(self: Self) -> Self: + return self + + async def __anext__(self) -> bytes: + part = await self.next() + if part is None: + raise StopAsyncIteration + return part + + async def next(self) -> Optional[bytes]: + item = await self.read() + if not item: + return None + return item + + async def read(self, *, decode: bool = False) -> bytes: + """Reads body part data. + + decode: Decodes data following by encoding + method from Content-Encoding header. If it missed + data remains untouched + """ + if self._at_eof: + return b"" + data = bytearray() + while not self._at_eof: + data.extend(await self.read_chunk(self.chunk_size)) + if decode: + return self.decode(data) + return data + + async def read_chunk(self, size: int = chunk_size) -> bytes: + """Reads body part content chunk of the specified size. + + size: chunk size + """ + if self._at_eof: + return b"" + if self._length: + chunk = await self._read_chunk_from_length(size) + else: + chunk = await self._read_chunk_from_stream(size) + + # For the case of base64 data, we must read a fragment of size with a + # remainder of 0 by dividing by 4 for string without symbols \n or \r + encoding = self.headers.get(CONTENT_TRANSFER_ENCODING) + if encoding and encoding.lower() == "base64": + stripped_chunk = b"".join(chunk.split()) + remainder = len(stripped_chunk) % 4 + + while remainder != 0 and not self.at_eof(): + over_chunk_size = 4 - remainder + over_chunk = b"" + + if self._prev_chunk: + over_chunk = self._prev_chunk[:over_chunk_size] + self._prev_chunk = self._prev_chunk[len(over_chunk) :] + + if len(over_chunk) != over_chunk_size: + over_chunk += await self._content.read(4 - len(over_chunk)) + + if not over_chunk: + self._at_eof = True + + stripped_chunk += b"".join(over_chunk.split()) + chunk += over_chunk + remainder = len(stripped_chunk) % 4 + + self._read_bytes += len(chunk) + if self._read_bytes == self._length: + self._at_eof = True + if self._at_eof: + clrf = await self._content.readline() + assert ( + b"\r\n" == clrf + ), "reader did not read all the data or it is malformed" + return chunk + + async def _read_chunk_from_length(self, size: int) -> bytes: + # Reads body part content chunk of the specified size. + # The body part must has Content-Length header with proper value. + assert self._length is not None, "Content-Length required for chunked read" + chunk_size = min(size, self._length - self._read_bytes) + chunk = await self._content.read(chunk_size) + if self._content.at_eof(): + self._at_eof = True + return chunk + + async def _read_chunk_from_stream(self, size: int) -> bytes: + # Reads content chunk of body part with unknown length. + # The Content-Length header for body part is not necessary. + assert ( + size >= self._boundary_len + ), "Chunk size must be greater or equal than boundary length + 2" + first_chunk = self._prev_chunk is None + if first_chunk: + self._prev_chunk = await self._content.read(size) + + chunk = b"" + # content.read() may return less than size, so we need to loop to ensure + # we have enough data to detect the boundary. + while len(chunk) < self._boundary_len: + chunk += await self._content.read(size) + self._content_eof += int(self._content.at_eof()) + assert self._content_eof < 3, "Reading after EOF" + if self._content_eof: + break + if len(chunk) > size: + self._content.unread_data(chunk[size:]) + chunk = chunk[:size] + + assert self._prev_chunk is not None + window = self._prev_chunk + chunk + sub = b"\r\n" + self._boundary + if first_chunk: + idx = window.find(sub) + else: + idx = window.find(sub, max(0, len(self._prev_chunk) - len(sub))) + if idx >= 0: + # pushing boundary back to content + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + self._content.unread_data(window[idx:]) + if size > idx: + self._prev_chunk = self._prev_chunk[:idx] + chunk = window[len(self._prev_chunk) : idx] + if not chunk: + self._at_eof = True + result = self._prev_chunk + self._prev_chunk = chunk + return result + + async def readline(self) -> bytes: + """Reads body part by line by line.""" + if self._at_eof: + return b"" + + if self._unread: + line = self._unread.popleft() + else: + line = await self._content.readline() + + if line.startswith(self._boundary): + # the very last boundary may not come with \r\n, + # so set single rules for everyone + sline = line.rstrip(b"\r\n") + boundary = self._boundary + last_boundary = self._boundary + b"--" + # ensure that we read exactly the boundary, not something alike + if sline == boundary or sline == last_boundary: + self._at_eof = True + self._unread.append(line) + return b"" + else: + next_line = await self._content.readline() + if next_line.startswith(self._boundary): + line = line[:-2] # strip CRLF but only once + self._unread.append(next_line) + + return line + + async def release(self) -> None: + """Like read(), but reads all the data to the void.""" + if self._at_eof: + return + while not self._at_eof: + await self.read_chunk(self.chunk_size) + + async def text(self, *, encoding: Optional[str] = None) -> str: + """Like read(), but assumes that body part contains text data.""" + data = await self.read(decode=True) + # see https://www.w3.org/TR/html5/forms.html#multipart/form-data-encoding-algorithm + # and https://dvcs.w3.org/hg/xhr/raw-file/tip/Overview.html#dom-xmlhttprequest-send + encoding = encoding or self.get_charset(default="utf-8") + return data.decode(encoding) + + async def json(self, *, encoding: Optional[str] = None) -> Optional[Dict[str, Any]]: + """Like read(), but assumes that body parts contains JSON data.""" + data = await self.read(decode=True) + if not data: + return None + encoding = encoding or self.get_charset(default="utf-8") + return cast(Dict[str, Any], json.loads(data.decode(encoding))) + + async def form(self, *, encoding: Optional[str] = None) -> List[Tuple[str, str]]: + """Like read(), but assumes that body parts contain form urlencoded data.""" + data = await self.read(decode=True) + if not data: + return [] + if encoding is not None: + real_encoding = encoding + else: + real_encoding = self.get_charset(default="utf-8") + try: + decoded_data = data.rstrip().decode(real_encoding) + except UnicodeDecodeError: + raise ValueError("data cannot be decoded with %s encoding" % real_encoding) + + return parse_qsl( + decoded_data, + keep_blank_values=True, + encoding=real_encoding, + ) + + def at_eof(self) -> bool: + """Returns True if the boundary was reached or False otherwise.""" + return self._at_eof + + def decode(self, data: bytes) -> bytes: + """Decodes data. + + Decoding is done according the specified Content-Encoding + or Content-Transfer-Encoding headers value. + """ + if CONTENT_TRANSFER_ENCODING in self.headers: + data = self._decode_content_transfer(data) + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.8 + if not self._is_form_data and CONTENT_ENCODING in self.headers: + return self._decode_content(data) + return data + + def _decode_content(self, data: bytes) -> bytes: + encoding = self.headers.get(CONTENT_ENCODING, "").lower() + if encoding == "identity": + return data + if encoding in {"deflate", "gzip"}: + return ZLibDecompressor( + encoding=encoding, + suppress_deflate_header=True, + ).decompress_sync(data) + + raise RuntimeError(f"unknown content encoding: {encoding}") + + def _decode_content_transfer(self, data: bytes) -> bytes: + encoding = self.headers.get(CONTENT_TRANSFER_ENCODING, "").lower() + + if encoding == "base64": + return base64.b64decode(data) + elif encoding == "quoted-printable": + return binascii.a2b_qp(data) + elif encoding in ("binary", "8bit", "7bit"): + return data + else: + raise RuntimeError(f"unknown content transfer encoding: {encoding}") + + def get_charset(self, default: str) -> str: + """Returns charset parameter from Content-Type header or default.""" + ctype = self.headers.get(CONTENT_TYPE, "") + mimetype = parse_mimetype(ctype) + return mimetype.parameters.get("charset", self._default_charset or default) + + @reify + def name(self) -> Optional[str]: + """Returns name specified in Content-Disposition header. + + If the header is missing or malformed, returns None. + """ + _, params = parse_content_disposition(self.headers.get(CONTENT_DISPOSITION)) + return content_disposition_filename(params, "name") + + @reify + def filename(self) -> Optional[str]: + """Returns filename specified in Content-Disposition header. + + Returns None if the header is missing or malformed. + """ + _, params = parse_content_disposition(self.headers.get(CONTENT_DISPOSITION)) + return content_disposition_filename(params, "filename") + + +@payload_type(BodyPartReader, order=Order.try_first) +class BodyPartReaderPayload(Payload): + _value: BodyPartReader + # _autoclose = False (inherited) - Streaming reader that may have resources + + def __init__(self, value: BodyPartReader, *args: Any, **kwargs: Any) -> None: + super().__init__(value, *args, **kwargs) + + params: Dict[str, str] = {} + if value.name is not None: + params["name"] = value.name + if value.filename is not None: + params["filename"] = value.filename + + if params: + self.set_content_disposition("attachment", True, **params) + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + raise TypeError("Unable to decode.") + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """Raises TypeError as body parts should be consumed via write(). + + This is intentional: BodyPartReader payloads are designed for streaming + large data (potentially gigabytes) and must be consumed only once via + the write() method to avoid memory exhaustion. They cannot be buffered + in memory for reuse. + """ + raise TypeError("Unable to read body part as bytes. Use write() to consume.") + + async def write(self, writer: Any) -> None: + field = self._value + chunk = await field.read_chunk(size=2**16) + while chunk: + await writer.write(field.decode(chunk)) + chunk = await field.read_chunk(size=2**16) + + +class MultipartReader: + """Multipart body reader.""" + + #: Response wrapper, used when multipart readers constructs from response. + response_wrapper_cls = MultipartResponseWrapper + #: Multipart reader class, used to handle multipart/* body parts. + #: None points to type(self) + multipart_reader_cls: Optional[Type["MultipartReader"]] = None + #: Body part reader class for non multipart/* content types. + part_reader_cls = BodyPartReader + + def __init__(self, headers: Mapping[str, str], content: StreamReader) -> None: + self._mimetype = parse_mimetype(headers[CONTENT_TYPE]) + assert self._mimetype.type == "multipart", "multipart/* content type expected" + if "boundary" not in self._mimetype.parameters: + raise ValueError( + "boundary missed for Content-Type: %s" % headers[CONTENT_TYPE] + ) + + self.headers = headers + self._boundary = ("--" + self._get_boundary()).encode() + self._content = content + self._default_charset: Optional[str] = None + self._last_part: Optional[Union["MultipartReader", BodyPartReader]] = None + self._at_eof = False + self._at_bof = True + self._unread: List[bytes] = [] + + def __aiter__(self: Self) -> Self: + return self + + async def __anext__( + self, + ) -> Optional[Union["MultipartReader", BodyPartReader]]: + part = await self.next() + if part is None: + raise StopAsyncIteration + return part + + @classmethod + def from_response( + cls, + response: "ClientResponse", + ) -> MultipartResponseWrapper: + """Constructs reader instance from HTTP response. + + :param response: :class:`~aiohttp.client.ClientResponse` instance + """ + obj = cls.response_wrapper_cls( + response, cls(response.headers, response.content) + ) + return obj + + def at_eof(self) -> bool: + """Returns True if the final boundary was reached, false otherwise.""" + return self._at_eof + + async def next( + self, + ) -> Optional[Union["MultipartReader", BodyPartReader]]: + """Emits the next multipart body part.""" + # So, if we're at BOF, we need to skip till the boundary. + if self._at_eof: + return None + await self._maybe_release_last_part() + if self._at_bof: + await self._read_until_first_boundary() + self._at_bof = False + else: + await self._read_boundary() + if self._at_eof: # we just read the last boundary, nothing to do there + return None + + part = await self.fetch_next_part() + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.6 + if ( + self._last_part is None + and self._mimetype.subtype == "form-data" + and isinstance(part, BodyPartReader) + ): + _, params = parse_content_disposition(part.headers.get(CONTENT_DISPOSITION)) + if params.get("name") == "_charset_": + # Longest encoding in https://encoding.spec.whatwg.org/encodings.json + # is 19 characters, so 32 should be more than enough for any valid encoding. + charset = await part.read_chunk(32) + if len(charset) > 31: + raise RuntimeError("Invalid default charset") + self._default_charset = charset.strip().decode() + part = await self.fetch_next_part() + self._last_part = part + return self._last_part + + async def release(self) -> None: + """Reads all the body parts to the void till the final boundary.""" + while not self._at_eof: + item = await self.next() + if item is None: + break + await item.release() + + async def fetch_next_part( + self, + ) -> Union["MultipartReader", BodyPartReader]: + """Returns the next body part reader.""" + headers = await self._read_headers() + return self._get_part_reader(headers) + + def _get_part_reader( + self, + headers: "CIMultiDictProxy[str]", + ) -> Union["MultipartReader", BodyPartReader]: + """Dispatches the response by the `Content-Type` header. + + Returns a suitable reader instance. + + :param dict headers: Response headers + """ + ctype = headers.get(CONTENT_TYPE, "") + mimetype = parse_mimetype(ctype) + + if mimetype.type == "multipart": + if self.multipart_reader_cls is None: + return type(self)(headers, self._content) + return self.multipart_reader_cls(headers, self._content) + else: + return self.part_reader_cls( + self._boundary, + headers, + self._content, + subtype=self._mimetype.subtype, + default_charset=self._default_charset, + ) + + def _get_boundary(self) -> str: + boundary = self._mimetype.parameters["boundary"] + if len(boundary) > 70: + raise ValueError("boundary %r is too long (70 chars max)" % boundary) + + return boundary + + async def _readline(self) -> bytes: + if self._unread: + return self._unread.pop() + return await self._content.readline() + + async def _read_until_first_boundary(self) -> None: + while True: + chunk = await self._readline() + if chunk == b"": + raise ValueError( + "Could not find starting boundary %r" % (self._boundary) + ) + chunk = chunk.rstrip() + if chunk == self._boundary: + return + elif chunk == self._boundary + b"--": + self._at_eof = True + return + + async def _read_boundary(self) -> None: + chunk = (await self._readline()).rstrip() + if chunk == self._boundary: + pass + elif chunk == self._boundary + b"--": + self._at_eof = True + epilogue = await self._readline() + next_line = await self._readline() + + # the epilogue is expected and then either the end of input or the + # parent multipart boundary, if the parent boundary is found then + # it should be marked as unread and handed to the parent for + # processing + if next_line[:2] == b"--": + self._unread.append(next_line) + # otherwise the request is likely missing an epilogue and both + # lines should be passed to the parent for processing + # (this handles the old behavior gracefully) + else: + self._unread.extend([next_line, epilogue]) + else: + raise ValueError(f"Invalid boundary {chunk!r}, expected {self._boundary!r}") + + async def _read_headers(self) -> "CIMultiDictProxy[str]": + lines = [] + while True: + chunk = await self._content.readline() + chunk = chunk.strip() + lines.append(chunk) + if not chunk: + break + parser = HeadersParser() + headers, raw_headers = parser.parse_headers(lines) + return headers + + async def _maybe_release_last_part(self) -> None: + """Ensures that the last read body part is read completely.""" + if self._last_part is not None: + if not self._last_part.at_eof(): + await self._last_part.release() + self._unread.extend(self._last_part._unread) + self._last_part = None + + +_Part = Tuple[Payload, str, str] + + +class MultipartWriter(Payload): + """Multipart body writer.""" + + _value: None + # _consumed = False (inherited) - Can be encoded multiple times + _autoclose = True # No file handles, just collects parts in memory + + def __init__(self, subtype: str = "mixed", boundary: Optional[str] = None) -> None: + boundary = boundary if boundary is not None else uuid.uuid4().hex + # The underlying Payload API demands a str (utf-8), not bytes, + # so we need to ensure we don't lose anything during conversion. + # As a result, require the boundary to be ASCII only. + # In both situations. + + try: + self._boundary = boundary.encode("ascii") + except UnicodeEncodeError: + raise ValueError("boundary should contain ASCII only chars") from None + ctype = f"multipart/{subtype}; boundary={self._boundary_value}" + + super().__init__(None, content_type=ctype) + + self._parts: List[_Part] = [] + self._is_form_data = subtype == "form-data" + + def __enter__(self) -> "MultipartWriter": + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + pass + + def __iter__(self) -> Iterator[_Part]: + return iter(self._parts) + + def __len__(self) -> int: + return len(self._parts) + + def __bool__(self) -> bool: + return True + + _valid_tchar_regex = re.compile(rb"\A[!#$%&'*+\-.^_`|~\w]+\Z") + _invalid_qdtext_char_regex = re.compile(rb"[\x00-\x08\x0A-\x1F\x7F]") + + @property + def _boundary_value(self) -> str: + """Wrap boundary parameter value in quotes, if necessary. + + Reads self.boundary and returns a unicode string. + """ + # Refer to RFCs 7231, 7230, 5234. + # + # parameter = token "=" ( token / quoted-string ) + # token = 1*tchar + # quoted-string = DQUOTE *( qdtext / quoted-pair ) DQUOTE + # qdtext = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text + # obs-text = %x80-FF + # quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text ) + # tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" + # / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" + # / DIGIT / ALPHA + # ; any VCHAR, except delimiters + # VCHAR = %x21-7E + value = self._boundary + if re.match(self._valid_tchar_regex, value): + return value.decode("ascii") # cannot fail + + if re.search(self._invalid_qdtext_char_regex, value): + raise ValueError("boundary value contains invalid characters") + + # escape %x5C and %x22 + quoted_value_content = value.replace(b"\\", b"\\\\") + quoted_value_content = quoted_value_content.replace(b'"', b'\\"') + + return '"' + quoted_value_content.decode("ascii") + '"' + + @property + def boundary(self) -> str: + return self._boundary.decode("ascii") + + def append(self, obj: Any, headers: Optional[Mapping[str, str]] = None) -> Payload: + if headers is None: + headers = CIMultiDict() + + if isinstance(obj, Payload): + obj.headers.update(headers) + return self.append_payload(obj) + else: + try: + payload = get_payload(obj, headers=headers) + except LookupError: + raise TypeError("Cannot create payload from %r" % obj) + else: + return self.append_payload(payload) + + def append_payload(self, payload: Payload) -> Payload: + """Adds a new body part to multipart writer.""" + encoding: Optional[str] = None + te_encoding: Optional[str] = None + if self._is_form_data: + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.7 + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.8 + assert ( + not {CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TRANSFER_ENCODING} + & payload.headers.keys() + ) + # Set default Content-Disposition in case user doesn't create one + if CONTENT_DISPOSITION not in payload.headers: + name = f"section-{len(self._parts)}" + payload.set_content_disposition("form-data", name=name) + else: + # compression + encoding = payload.headers.get(CONTENT_ENCODING, "").lower() + if encoding and encoding not in ("deflate", "gzip", "identity"): + raise RuntimeError(f"unknown content encoding: {encoding}") + if encoding == "identity": + encoding = None + + # te encoding + te_encoding = payload.headers.get(CONTENT_TRANSFER_ENCODING, "").lower() + if te_encoding not in ("", "base64", "quoted-printable", "binary"): + raise RuntimeError(f"unknown content transfer encoding: {te_encoding}") + if te_encoding == "binary": + te_encoding = None + + # size + size = payload.size + if size is not None and not (encoding or te_encoding): + payload.headers[CONTENT_LENGTH] = str(size) + + self._parts.append((payload, encoding, te_encoding)) # type: ignore[arg-type] + return payload + + def append_json( + self, obj: Any, headers: Optional[Mapping[str, str]] = None + ) -> Payload: + """Helper to append JSON part.""" + if headers is None: + headers = CIMultiDict() + + return self.append_payload(JsonPayload(obj, headers=headers)) + + def append_form( + self, + obj: Union[Sequence[Tuple[str, str]], Mapping[str, str]], + headers: Optional[Mapping[str, str]] = None, + ) -> Payload: + """Helper to append form urlencoded part.""" + assert isinstance(obj, (Sequence, Mapping)) + + if headers is None: + headers = CIMultiDict() + + if isinstance(obj, Mapping): + obj = list(obj.items()) + data = urlencode(obj, doseq=True) + + return self.append_payload( + StringPayload( + data, headers=headers, content_type="application/x-www-form-urlencoded" + ) + ) + + @property + def size(self) -> Optional[int]: + """Size of the payload.""" + total = 0 + for part, encoding, te_encoding in self._parts: + part_size = part.size + if encoding or te_encoding or part_size is None: + return None + + total += int( + 2 + + len(self._boundary) + + 2 + + part_size # b'--'+self._boundary+b'\r\n' + + len(part._binary_headers) + + 2 # b'\r\n' + ) + + total += 2 + len(self._boundary) + 4 # b'--'+self._boundary+b'--\r\n' + return total + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """Return string representation of the multipart data. + + WARNING: This method may do blocking I/O if parts contain file payloads. + It should not be called in the event loop. Use as_bytes().decode() instead. + """ + return "".join( + "--" + + self.boundary + + "\r\n" + + part._binary_headers.decode(encoding, errors) + + part.decode() + for part, _e, _te in self._parts + ) + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """Return bytes representation of the multipart data. + + This method is async-safe and calls as_bytes on underlying payloads. + """ + parts: List[bytes] = [] + + # Process each part + for part, _e, _te in self._parts: + # Add boundary + parts.append(b"--" + self._boundary + b"\r\n") + + # Add headers + parts.append(part._binary_headers) + + # Add payload content using as_bytes for async safety + part_bytes = await part.as_bytes(encoding, errors) + parts.append(part_bytes) + + # Add trailing CRLF + parts.append(b"\r\n") + + # Add closing boundary + parts.append(b"--" + self._boundary + b"--\r\n") + + return b"".join(parts) + + async def write(self, writer: Any, close_boundary: bool = True) -> None: + """Write body.""" + for part, encoding, te_encoding in self._parts: + if self._is_form_data: + # https://datatracker.ietf.org/doc/html/rfc7578#section-4.2 + assert CONTENT_DISPOSITION in part.headers + assert "name=" in part.headers[CONTENT_DISPOSITION] + + await writer.write(b"--" + self._boundary + b"\r\n") + await writer.write(part._binary_headers) + + if encoding or te_encoding: + w = MultipartPayloadWriter(writer) + if encoding: + w.enable_compression(encoding) + if te_encoding: + w.enable_encoding(te_encoding) + await part.write(w) # type: ignore[arg-type] + await w.write_eof() + else: + await part.write(writer) + + await writer.write(b"\r\n") + + if close_boundary: + await writer.write(b"--" + self._boundary + b"--\r\n") + + async def close(self) -> None: + """ + Close all part payloads that need explicit closing. + + IMPORTANT: This method must not await anything that might not finish + immediately, as it may be called during cleanup/cancellation. Schedule + any long-running operations without awaiting them. + """ + if self._consumed: + return + self._consumed = True + + # Close all parts that need explicit closing + # We catch and log exceptions to ensure all parts get a chance to close + # we do not use asyncio.gather() here because we are not allowed + # to suspend given we may be called during cleanup + for idx, (part, _, _) in enumerate(self._parts): + if not part.autoclose and not part.consumed: + try: + await part.close() + except Exception as exc: + internal_logger.error( + "Failed to close multipart part %d: %s", idx, exc, exc_info=True + ) + + +class MultipartPayloadWriter: + def __init__(self, writer: Any) -> None: + self._writer = writer + self._encoding: Optional[str] = None + self._compress: Optional[ZLibCompressor] = None + self._encoding_buffer: Optional[bytearray] = None + + def enable_encoding(self, encoding: str) -> None: + if encoding == "base64": + self._encoding = encoding + self._encoding_buffer = bytearray() + elif encoding == "quoted-printable": + self._encoding = "quoted-printable" + + def enable_compression( + self, encoding: str = "deflate", strategy: Optional[int] = None + ) -> None: + self._compress = ZLibCompressor( + encoding=encoding, + suppress_deflate_header=True, + strategy=strategy, + ) + + async def write_eof(self) -> None: + if self._compress is not None: + chunk = self._compress.flush() + if chunk: + self._compress = None + await self.write(chunk) + + if self._encoding == "base64": + if self._encoding_buffer: + await self._writer.write(base64.b64encode(self._encoding_buffer)) + + async def write(self, chunk: bytes) -> None: + if self._compress is not None: + if chunk: + chunk = await self._compress.compress(chunk) + if not chunk: + return + + if self._encoding == "base64": + buf = self._encoding_buffer + assert buf is not None + buf.extend(chunk) + + if buf: + div, mod = divmod(len(buf), 3) + enc_chunk, self._encoding_buffer = (buf[: div * 3], buf[div * 3 :]) + if enc_chunk: + b64chunk = base64.b64encode(enc_chunk) + await self._writer.write(b64chunk) + elif self._encoding == "quoted-printable": + await self._writer.write(binascii.b2a_qp(chunk)) + else: + await self._writer.write(chunk) diff --git a/lib/python3.1/site-packages/aiohttp/payload.py b/lib/python3.1/site-packages/aiohttp/payload.py new file mode 100644 index 0000000000000000000000000000000000000000..5b88fa094050fcef280050b54429f6bd43b166d3 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/payload.py @@ -0,0 +1,1120 @@ +import asyncio +import enum +import io +import json +import mimetypes +import os +import sys +import warnings +from abc import ABC, abstractmethod +from collections.abc import Iterable +from itertools import chain +from typing import ( + IO, + TYPE_CHECKING, + Any, + Dict, + Final, + List, + Optional, + Set, + TextIO, + Tuple, + Type, + Union, +) + +from multidict import CIMultiDict + +from . import hdrs +from .abc import AbstractStreamWriter +from .helpers import ( + _SENTINEL, + content_disposition_header, + guess_filename, + parse_mimetype, + sentinel, +) +from .streams import StreamReader +from .typedefs import JSONEncoder, _CIMultiDict + +__all__ = ( + "PAYLOAD_REGISTRY", + "get_payload", + "payload_type", + "Payload", + "BytesPayload", + "StringPayload", + "IOBasePayload", + "BytesIOPayload", + "BufferedReaderPayload", + "TextIOPayload", + "StringIOPayload", + "JsonPayload", + "AsyncIterablePayload", +) + +TOO_LARGE_BYTES_BODY: Final[int] = 2**20 # 1 MB +READ_SIZE: Final[int] = 2**16 # 64 KB +_CLOSE_FUTURES: Set[asyncio.Future[None]] = set() + + +class LookupError(Exception): + """Raised when no payload factory is found for the given data type.""" + + +class Order(str, enum.Enum): + normal = "normal" + try_first = "try_first" + try_last = "try_last" + + +def get_payload(data: Any, *args: Any, **kwargs: Any) -> "Payload": + return PAYLOAD_REGISTRY.get(data, *args, **kwargs) + + +def register_payload( + factory: Type["Payload"], type: Any, *, order: Order = Order.normal +) -> None: + PAYLOAD_REGISTRY.register(factory, type, order=order) + + +class payload_type: + def __init__(self, type: Any, *, order: Order = Order.normal) -> None: + self.type = type + self.order = order + + def __call__(self, factory: Type["Payload"]) -> Type["Payload"]: + register_payload(factory, self.type, order=self.order) + return factory + + +PayloadType = Type["Payload"] +_PayloadRegistryItem = Tuple[PayloadType, Any] + + +class PayloadRegistry: + """Payload registry. + + note: we need zope.interface for more efficient adapter search + """ + + __slots__ = ("_first", "_normal", "_last", "_normal_lookup") + + def __init__(self) -> None: + self._first: List[_PayloadRegistryItem] = [] + self._normal: List[_PayloadRegistryItem] = [] + self._last: List[_PayloadRegistryItem] = [] + self._normal_lookup: Dict[Any, PayloadType] = {} + + def get( + self, + data: Any, + *args: Any, + _CHAIN: "Type[chain[_PayloadRegistryItem]]" = chain, + **kwargs: Any, + ) -> "Payload": + if self._first: + for factory, type_ in self._first: + if isinstance(data, type_): + return factory(data, *args, **kwargs) + # Try the fast lookup first + if lookup_factory := self._normal_lookup.get(type(data)): + return lookup_factory(data, *args, **kwargs) + # Bail early if its already a Payload + if isinstance(data, Payload): + return data + # Fallback to the slower linear search + for factory, type_ in _CHAIN(self._normal, self._last): + if isinstance(data, type_): + return factory(data, *args, **kwargs) + raise LookupError() + + def register( + self, factory: PayloadType, type: Any, *, order: Order = Order.normal + ) -> None: + if order is Order.try_first: + self._first.append((factory, type)) + elif order is Order.normal: + self._normal.append((factory, type)) + if isinstance(type, Iterable): + for t in type: + self._normal_lookup[t] = factory + else: + self._normal_lookup[type] = factory + elif order is Order.try_last: + self._last.append((factory, type)) + else: + raise ValueError(f"Unsupported order {order!r}") + + +class Payload(ABC): + + _default_content_type: str = "application/octet-stream" + _size: Optional[int] = None + _consumed: bool = False # Default: payload has not been consumed yet + _autoclose: bool = False # Default: assume resource needs explicit closing + + def __init__( + self, + value: Any, + headers: Optional[ + Union[_CIMultiDict, Dict[str, str], Iterable[Tuple[str, str]]] + ] = None, + content_type: Union[str, None, _SENTINEL] = sentinel, + filename: Optional[str] = None, + encoding: Optional[str] = None, + **kwargs: Any, + ) -> None: + self._encoding = encoding + self._filename = filename + self._headers: _CIMultiDict = CIMultiDict() + self._value = value + if content_type is not sentinel and content_type is not None: + self._headers[hdrs.CONTENT_TYPE] = content_type + elif self._filename is not None: + if sys.version_info >= (3, 13): + guesser = mimetypes.guess_file_type + else: + guesser = mimetypes.guess_type + content_type = guesser(self._filename)[0] + if content_type is None: + content_type = self._default_content_type + self._headers[hdrs.CONTENT_TYPE] = content_type + else: + self._headers[hdrs.CONTENT_TYPE] = self._default_content_type + if headers: + self._headers.update(headers) + + @property + def size(self) -> Optional[int]: + """Size of the payload in bytes. + + Returns the number of bytes that will be transmitted when the payload + is written. For string payloads, this is the size after encoding to bytes, + not the length of the string. + """ + return self._size + + @property + def filename(self) -> Optional[str]: + """Filename of the payload.""" + return self._filename + + @property + def headers(self) -> _CIMultiDict: + """Custom item headers""" + return self._headers + + @property + def _binary_headers(self) -> bytes: + return ( + "".join([k + ": " + v + "\r\n" for k, v in self.headers.items()]).encode( + "utf-8" + ) + + b"\r\n" + ) + + @property + def encoding(self) -> Optional[str]: + """Payload encoding""" + return self._encoding + + @property + def content_type(self) -> str: + """Content type""" + return self._headers[hdrs.CONTENT_TYPE] + + @property + def consumed(self) -> bool: + """Whether the payload has been consumed and cannot be reused.""" + return self._consumed + + @property + def autoclose(self) -> bool: + """ + Whether the payload can close itself automatically. + + Returns True if the payload has no file handles or resources that need + explicit closing. If False, callers must await close() to release resources. + """ + return self._autoclose + + def set_content_disposition( + self, + disptype: str, + quote_fields: bool = True, + _charset: str = "utf-8", + **params: Any, + ) -> None: + """Sets ``Content-Disposition`` header.""" + self._headers[hdrs.CONTENT_DISPOSITION] = content_disposition_header( + disptype, quote_fields=quote_fields, _charset=_charset, **params + ) + + @abstractmethod + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """ + Return string representation of the value. + + This is named decode() to allow compatibility with bytes objects. + """ + + @abstractmethod + async def write(self, writer: AbstractStreamWriter) -> None: + """ + Write payload to the writer stream. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + + This is a legacy method that writes the entire payload without length constraints. + + Important: + For new implementations, use write_with_length() instead of this method. + This method is maintained for backwards compatibility and will eventually + delegate to write_with_length(writer, None) in all implementations. + + All payload subclasses must override this method for backwards compatibility, + but new code should use write_with_length for more flexibility and control. + + """ + + # write_with_length is new in aiohttp 3.12 + # it should be overridden by subclasses + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: Optional[int] + ) -> None: + """ + Write payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This method allows writing payload content with a specific length constraint, + which is particularly useful for HTTP responses with Content-Length header. + + Note: + This is the base implementation that provides backwards compatibility + for subclasses that don't override this method. Specific payload types + should override this method to implement proper length-constrained writing. + + """ + # Backwards compatibility for subclasses that don't override this method + # and for the default implementation + await self.write(writer) + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This is a convenience method that calls decode() and encodes the result + to bytes using the specified encoding. + """ + # Use instance encoding if available, otherwise use parameter + actual_encoding = self._encoding or encoding + return self.decode(actual_encoding, errors).encode(actual_encoding) + + def _close(self) -> None: + """ + Async safe synchronous close operations for backwards compatibility. + + This method exists only for backwards compatibility with code that + needs to clean up payloads synchronously. In the future, we will + drop this method and only support the async close() method. + + WARNING: This method must be safe to call from within the event loop + without blocking. Subclasses should not perform any blocking I/O here. + + WARNING: This method must be called from within an event loop for + certain payload types (e.g., IOBasePayload). Calling it outside an + event loop may raise RuntimeError. + """ + # This is a no-op by default, but subclasses can override it + # for non-blocking cleanup operations. + + async def close(self) -> None: + """ + Close the payload if it holds any resources. + + IMPORTANT: This method must not await anything that might not finish + immediately, as it may be called during cleanup/cancellation. Schedule + any long-running operations without awaiting them. + + In the future, this will be the only close method supported. + """ + self._close() + + +class BytesPayload(Payload): + _value: bytes + # _consumed = False (inherited) - Bytes are immutable and can be reused + _autoclose = True # No file handle, just bytes in memory + + def __init__( + self, value: Union[bytes, bytearray, memoryview], *args: Any, **kwargs: Any + ) -> None: + if "content_type" not in kwargs: + kwargs["content_type"] = "application/octet-stream" + + super().__init__(value, *args, **kwargs) + + if isinstance(value, memoryview): + self._size = value.nbytes + elif isinstance(value, (bytes, bytearray)): + self._size = len(value) + else: + raise TypeError(f"value argument must be byte-ish, not {type(value)!r}") + + if self._size > TOO_LARGE_BYTES_BODY: + kwargs = {"source": self} + warnings.warn( + "Sending a large body directly with raw bytes might" + " lock the event loop. You should probably pass an " + "io.BytesIO object instead", + ResourceWarning, + **kwargs, + ) + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + return self._value.decode(encoding, errors) + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method returns the raw bytes content of the payload. + It is equivalent to accessing the _value attribute directly. + """ + return self._value + + async def write(self, writer: AbstractStreamWriter) -> None: + """ + Write the entire bytes payload to the writer stream. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + + This method writes the entire bytes content without any length constraint. + + Note: + For new implementations that need length control, use write_with_length(). + This method is maintained for backwards compatibility and is equivalent + to write_with_length(writer, None). + + """ + await writer.write(self._value) + + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: Optional[int] + ) -> None: + """ + Write bytes payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This method writes either the entire byte sequence or a slice of it + up to the specified content_length. For BytesPayload, this operation + is performed efficiently using array slicing. + + """ + if content_length is not None: + await writer.write(self._value[:content_length]) + else: + await writer.write(self._value) + + +class StringPayload(BytesPayload): + def __init__( + self, + value: str, + *args: Any, + encoding: Optional[str] = None, + content_type: Optional[str] = None, + **kwargs: Any, + ) -> None: + + if encoding is None: + if content_type is None: + real_encoding = "utf-8" + content_type = "text/plain; charset=utf-8" + else: + mimetype = parse_mimetype(content_type) + real_encoding = mimetype.parameters.get("charset", "utf-8") + else: + if content_type is None: + content_type = "text/plain; charset=%s" % encoding + real_encoding = encoding + + super().__init__( + value.encode(real_encoding), + encoding=real_encoding, + content_type=content_type, + *args, + **kwargs, + ) + + +class StringIOPayload(StringPayload): + def __init__(self, value: IO[str], *args: Any, **kwargs: Any) -> None: + super().__init__(value.read(), *args, **kwargs) + + +class IOBasePayload(Payload): + _value: io.IOBase + # _consumed = False (inherited) - File can be re-read from the same position + _start_position: Optional[int] = None + # _autoclose = False (inherited) - Has file handle that needs explicit closing + + def __init__( + self, value: IO[Any], disposition: str = "attachment", *args: Any, **kwargs: Any + ) -> None: + if "filename" not in kwargs: + kwargs["filename"] = guess_filename(value) + + super().__init__(value, *args, **kwargs) + + if self._filename is not None and disposition is not None: + if hdrs.CONTENT_DISPOSITION not in self.headers: + self.set_content_disposition(disposition, filename=self._filename) + + def _set_or_restore_start_position(self) -> None: + """Set or restore the start position of the file-like object.""" + if self._start_position is None: + try: + self._start_position = self._value.tell() + except (OSError, AttributeError): + self._consumed = True # Cannot seek, mark as consumed + return + try: + self._value.seek(self._start_position) + except (OSError, AttributeError): + # Failed to seek back - mark as consumed since we've already read + self._consumed = True + + def _read_and_available_len( + self, remaining_content_len: Optional[int] + ) -> Tuple[Optional[int], bytes]: + """ + Read the file-like object and return both its total size and the first chunk. + + Args: + remaining_content_len: Optional limit on how many bytes to read in this operation. + If None, READ_SIZE will be used as the default chunk size. + + Returns: + A tuple containing: + - The total size of the remaining unread content (None if size cannot be determined) + - The first chunk of bytes read from the file object + + This method is optimized to perform both size calculation and initial read + in a single operation, which is executed in a single executor job to minimize + context switches and file operations when streaming content. + + """ + self._set_or_restore_start_position() + size = self.size # Call size only once since it does I/O + return size, self._value.read( + min(READ_SIZE, size or READ_SIZE, remaining_content_len or READ_SIZE) + ) + + def _read(self, remaining_content_len: Optional[int]) -> bytes: + """ + Read a chunk of data from the file-like object. + + Args: + remaining_content_len: Optional maximum number of bytes to read. + If None, READ_SIZE will be used as the default chunk size. + + Returns: + A chunk of bytes read from the file object, respecting the + remaining_content_len limit if specified. + + This method is used for subsequent reads during streaming after + the initial _read_and_available_len call has been made. + + """ + return self._value.read(remaining_content_len or READ_SIZE) # type: ignore[no-any-return] + + @property + def size(self) -> Optional[int]: + """ + Size of the payload in bytes. + + Returns the total size of the payload content from the initial position. + This ensures consistent Content-Length for requests, including 307/308 redirects + where the same payload instance is reused. + + Returns None if the size cannot be determined (e.g., for unseekable streams). + """ + try: + # Store the start position on first access. + # This is critical when the same payload instance is reused (e.g., 307/308 + # redirects). Without storing the initial position, after the payload is + # read once, the file position would be at EOF, which would cause the + # size calculation to return 0 (file_size - EOF position). + # By storing the start position, we ensure the size calculation always + # returns the correct total size for any subsequent use. + if self._start_position is None: + self._start_position = self._value.tell() + + # Return the total size from the start position + # This ensures Content-Length is correct even after reading + return os.fstat(self._value.fileno()).st_size - self._start_position + except (AttributeError, OSError): + return None + + async def write(self, writer: AbstractStreamWriter) -> None: + """ + Write the entire file-like payload to the writer stream. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + + This method writes the entire file content without any length constraint. + It delegates to write_with_length() with no length limit for implementation + consistency. + + Note: + For new implementations that need length control, use write_with_length() directly. + This method is maintained for backwards compatibility with existing code. + + """ + await self.write_with_length(writer, None) + + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: Optional[int] + ) -> None: + """ + Write file-like payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This method implements optimized streaming of file content with length constraints: + + 1. File reading is performed in a thread pool to avoid blocking the event loop + 2. Content is read and written in chunks to maintain memory efficiency + 3. Writing stops when either: + - All available file content has been written (when size is known) + - The specified content_length has been reached + 4. File resources are properly closed even if the operation is cancelled + + The implementation carefully handles both known-size and unknown-size payloads, + as well as constrained and unconstrained content lengths. + + """ + loop = asyncio.get_running_loop() + total_written_len = 0 + remaining_content_len = content_length + + # Get initial data and available length + available_len, chunk = await loop.run_in_executor( + None, self._read_and_available_len, remaining_content_len + ) + # Process data chunks until done + while chunk: + chunk_len = len(chunk) + + # Write data with or without length constraint + if remaining_content_len is None: + await writer.write(chunk) + else: + await writer.write(chunk[:remaining_content_len]) + remaining_content_len -= chunk_len + + total_written_len += chunk_len + + # Check if we're done writing + if self._should_stop_writing( + available_len, total_written_len, remaining_content_len + ): + return + + # Read next chunk + chunk = await loop.run_in_executor( + None, + self._read, + ( + min(READ_SIZE, remaining_content_len) + if remaining_content_len is not None + else READ_SIZE + ), + ) + + def _should_stop_writing( + self, + available_len: Optional[int], + total_written_len: int, + remaining_content_len: Optional[int], + ) -> bool: + """ + Determine if we should stop writing data. + + Args: + available_len: Known size of the payload if available (None if unknown) + total_written_len: Number of bytes already written + remaining_content_len: Remaining bytes to be written for content-length limited responses + + Returns: + True if we should stop writing data, based on either: + - Having written all available data (when size is known) + - Having written all requested content (when content-length is specified) + + """ + return (available_len is not None and total_written_len >= available_len) or ( + remaining_content_len is not None and remaining_content_len <= 0 + ) + + def _close(self) -> None: + """ + Async safe synchronous close operations for backwards compatibility. + + This method exists only for backwards + compatibility. Use the async close() method instead. + + WARNING: This method MUST be called from within an event loop. + Calling it outside an event loop will raise RuntimeError. + """ + # Skip if already consumed + if self._consumed: + return + self._consumed = True # Mark as consumed to prevent further writes + # Schedule file closing without awaiting to prevent cancellation issues + loop = asyncio.get_running_loop() + close_future = loop.run_in_executor(None, self._value.close) + # Hold a strong reference to the future to prevent it from being + # garbage collected before it completes. + _CLOSE_FUTURES.add(close_future) + close_future.add_done_callback(_CLOSE_FUTURES.remove) + + async def close(self) -> None: + """ + Close the payload if it holds any resources. + + IMPORTANT: This method must not await anything that might not finish + immediately, as it may be called during cleanup/cancellation. Schedule + any long-running operations without awaiting them. + """ + self._close() + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """ + Return string representation of the value. + + WARNING: This method does blocking I/O and should not be called in the event loop. + """ + return self._read_all().decode(encoding, errors) + + def _read_all(self) -> bytes: + """Read the entire file-like object and return its content as bytes.""" + self._set_or_restore_start_position() + # Use readlines() to ensure we get all content + return b"".join(self._value.readlines()) + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method reads the entire file content and returns it as bytes. + It is equivalent to reading the file-like object directly. + The file reading is performed in an executor to avoid blocking the event loop. + """ + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, self._read_all) + + +class TextIOPayload(IOBasePayload): + _value: io.TextIOBase + # _autoclose = False (inherited) - Has text file handle that needs explicit closing + + def __init__( + self, + value: TextIO, + *args: Any, + encoding: Optional[str] = None, + content_type: Optional[str] = None, + **kwargs: Any, + ) -> None: + + if encoding is None: + if content_type is None: + encoding = "utf-8" + content_type = "text/plain; charset=utf-8" + else: + mimetype = parse_mimetype(content_type) + encoding = mimetype.parameters.get("charset", "utf-8") + else: + if content_type is None: + content_type = "text/plain; charset=%s" % encoding + + super().__init__( + value, + content_type=content_type, + encoding=encoding, + *args, + **kwargs, + ) + + def _read_and_available_len( + self, remaining_content_len: Optional[int] + ) -> Tuple[Optional[int], bytes]: + """ + Read the text file-like object and return both its total size and the first chunk. + + Args: + remaining_content_len: Optional limit on how many bytes to read in this operation. + If None, READ_SIZE will be used as the default chunk size. + + Returns: + A tuple containing: + - The total size of the remaining unread content (None if size cannot be determined) + - The first chunk of bytes read from the file object, encoded using the payload's encoding + + This method is optimized to perform both size calculation and initial read + in a single operation, which is executed in a single executor job to minimize + context switches and file operations when streaming content. + + Note: + TextIOPayload handles encoding of the text content before writing it + to the stream. If no encoding is specified, UTF-8 is used as the default. + + """ + self._set_or_restore_start_position() + size = self.size + chunk = self._value.read( + min(READ_SIZE, size or READ_SIZE, remaining_content_len or READ_SIZE) + ) + return size, chunk.encode(self._encoding) if self._encoding else chunk.encode() + + def _read(self, remaining_content_len: Optional[int]) -> bytes: + """ + Read a chunk of data from the text file-like object. + + Args: + remaining_content_len: Optional maximum number of bytes to read. + If None, READ_SIZE will be used as the default chunk size. + + Returns: + A chunk of bytes read from the file object and encoded using the payload's + encoding. The data is automatically converted from text to bytes. + + This method is used for subsequent reads during streaming after + the initial _read_and_available_len call has been made. It properly + handles text encoding, converting the text content to bytes using + the specified encoding (or UTF-8 if none was provided). + + """ + chunk = self._value.read(remaining_content_len or READ_SIZE) + return chunk.encode(self._encoding) if self._encoding else chunk.encode() + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """ + Return string representation of the value. + + WARNING: This method does blocking I/O and should not be called in the event loop. + """ + self._set_or_restore_start_position() + return self._value.read() + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method reads the entire text file content and returns it as bytes. + It encodes the text content using the specified encoding. + The file reading is performed in an executor to avoid blocking the event loop. + """ + loop = asyncio.get_running_loop() + + # Use instance encoding if available, otherwise use parameter + actual_encoding = self._encoding or encoding + + def _read_and_encode() -> bytes: + self._set_or_restore_start_position() + # TextIO read() always returns the full content + return self._value.read().encode(actual_encoding, errors) + + return await loop.run_in_executor(None, _read_and_encode) + + +class BytesIOPayload(IOBasePayload): + _value: io.BytesIO + _size: int # Always initialized in __init__ + _autoclose = True # BytesIO is in-memory, safe to auto-close + + def __init__(self, value: io.BytesIO, *args: Any, **kwargs: Any) -> None: + super().__init__(value, *args, **kwargs) + # Calculate size once during initialization + self._size = len(self._value.getbuffer()) - self._value.tell() + + @property + def size(self) -> int: + """Size of the payload in bytes. + + Returns the number of bytes in the BytesIO buffer that will be transmitted. + This is calculated once during initialization for efficiency. + """ + return self._size + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + self._set_or_restore_start_position() + return self._value.read().decode(encoding, errors) + + async def write(self, writer: AbstractStreamWriter) -> None: + return await self.write_with_length(writer, None) + + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: Optional[int] + ) -> None: + """ + Write BytesIO payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This implementation is specifically optimized for BytesIO objects: + + 1. Reads content in chunks to maintain memory efficiency + 2. Yields control back to the event loop periodically to prevent blocking + when dealing with large BytesIO objects + 3. Respects content_length constraints when specified + 4. Properly cleans up by closing the BytesIO object when done or on error + + The periodic yielding to the event loop is important for maintaining + responsiveness when processing large in-memory buffers. + + """ + self._set_or_restore_start_position() + loop_count = 0 + remaining_bytes = content_length + while chunk := self._value.read(READ_SIZE): + if loop_count > 0: + # Avoid blocking the event loop + # if they pass a large BytesIO object + # and we are not in the first iteration + # of the loop + await asyncio.sleep(0) + if remaining_bytes is None: + await writer.write(chunk) + else: + await writer.write(chunk[:remaining_bytes]) + remaining_bytes -= len(chunk) + if remaining_bytes <= 0: + return + loop_count += 1 + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method reads the entire BytesIO content and returns it as bytes. + It is equivalent to accessing the _value attribute directly. + """ + self._set_or_restore_start_position() + return self._value.read() + + async def close(self) -> None: + """ + Close the BytesIO payload. + + This does nothing since BytesIO is in-memory and does not require explicit closing. + """ + + +class BufferedReaderPayload(IOBasePayload): + _value: io.BufferedIOBase + # _autoclose = False (inherited) - Has buffered file handle that needs explicit closing + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + self._set_or_restore_start_position() + return self._value.read().decode(encoding, errors) + + +class JsonPayload(BytesPayload): + def __init__( + self, + value: Any, + encoding: str = "utf-8", + content_type: str = "application/json", + dumps: JSONEncoder = json.dumps, + *args: Any, + **kwargs: Any, + ) -> None: + + super().__init__( + dumps(value).encode(encoding), + content_type=content_type, + encoding=encoding, + *args, + **kwargs, + ) + + +if TYPE_CHECKING: + from typing import AsyncIterable, AsyncIterator + + _AsyncIterator = AsyncIterator[bytes] + _AsyncIterable = AsyncIterable[bytes] +else: + from collections.abc import AsyncIterable, AsyncIterator + + _AsyncIterator = AsyncIterator + _AsyncIterable = AsyncIterable + + +class AsyncIterablePayload(Payload): + + _iter: Optional[_AsyncIterator] = None + _value: _AsyncIterable + _cached_chunks: Optional[List[bytes]] = None + # _consumed stays False to allow reuse with cached content + _autoclose = True # Iterator doesn't need explicit closing + + def __init__(self, value: _AsyncIterable, *args: Any, **kwargs: Any) -> None: + if not isinstance(value, AsyncIterable): + raise TypeError( + "value argument must support " + "collections.abc.AsyncIterable interface, " + "got {!r}".format(type(value)) + ) + + if "content_type" not in kwargs: + kwargs["content_type"] = "application/octet-stream" + + super().__init__(value, *args, **kwargs) + + self._iter = value.__aiter__() + + async def write(self, writer: AbstractStreamWriter) -> None: + """ + Write the entire async iterable payload to the writer stream. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + + This method iterates through the async iterable and writes each chunk + to the writer without any length constraint. + + Note: + For new implementations that need length control, use write_with_length() directly. + This method is maintained for backwards compatibility with existing code. + + """ + await self.write_with_length(writer, None) + + async def write_with_length( + self, writer: AbstractStreamWriter, content_length: Optional[int] + ) -> None: + """ + Write async iterable payload with a specific content length constraint. + + Args: + writer: An AbstractStreamWriter instance that handles the actual writing + content_length: Maximum number of bytes to write (None for unlimited) + + This implementation handles streaming of async iterable content with length constraints: + + 1. If cached chunks are available, writes from them + 2. Otherwise iterates through the async iterable one chunk at a time + 3. Respects content_length constraints when specified + 4. Does NOT generate cache - that's done by as_bytes() + + """ + # If we have cached chunks, use them + if self._cached_chunks is not None: + remaining_bytes = content_length + for chunk in self._cached_chunks: + if remaining_bytes is None: + await writer.write(chunk) + elif remaining_bytes > 0: + await writer.write(chunk[:remaining_bytes]) + remaining_bytes -= len(chunk) + else: + break + return + + # If iterator is exhausted and we don't have cached chunks, nothing to write + if self._iter is None: + return + + # Stream from the iterator + remaining_bytes = content_length + + try: + while True: + if sys.version_info >= (3, 10): + chunk = await anext(self._iter) + else: + chunk = await self._iter.__anext__() + if remaining_bytes is None: + await writer.write(chunk) + # If we have a content length limit + elif remaining_bytes > 0: + await writer.write(chunk[:remaining_bytes]) + remaining_bytes -= len(chunk) + # We still want to exhaust the iterator even + # if we have reached the content length limit + # since the file handle may not get closed by + # the iterator if we don't do this + except StopAsyncIteration: + # Iterator is exhausted + self._iter = None + self._consumed = True # Mark as consumed when streamed without caching + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + """Decode the payload content as a string if cached chunks are available.""" + if self._cached_chunks is not None: + return b"".join(self._cached_chunks).decode(encoding, errors) + raise TypeError("Unable to decode - content not cached. Call as_bytes() first.") + + async def as_bytes(self, encoding: str = "utf-8", errors: str = "strict") -> bytes: + """ + Return bytes representation of the value. + + This method reads the entire async iterable content and returns it as bytes. + It generates and caches the chunks for future reuse. + """ + # If we have cached chunks, return them joined + if self._cached_chunks is not None: + return b"".join(self._cached_chunks) + + # If iterator is exhausted and no cache, return empty + if self._iter is None: + return b"" + + # Read all chunks and cache them + chunks: List[bytes] = [] + async for chunk in self._iter: + chunks.append(chunk) + + # Iterator is exhausted, cache the chunks + self._iter = None + self._cached_chunks = chunks + # Keep _consumed as False to allow reuse with cached chunks + + return b"".join(chunks) + + +class StreamReaderPayload(AsyncIterablePayload): + def __init__(self, value: StreamReader, *args: Any, **kwargs: Any) -> None: + super().__init__(value.iter_any(), *args, **kwargs) + + +PAYLOAD_REGISTRY = PayloadRegistry() +PAYLOAD_REGISTRY.register(BytesPayload, (bytes, bytearray, memoryview)) +PAYLOAD_REGISTRY.register(StringPayload, str) +PAYLOAD_REGISTRY.register(StringIOPayload, io.StringIO) +PAYLOAD_REGISTRY.register(TextIOPayload, io.TextIOBase) +PAYLOAD_REGISTRY.register(BytesIOPayload, io.BytesIO) +PAYLOAD_REGISTRY.register(BufferedReaderPayload, (io.BufferedReader, io.BufferedRandom)) +PAYLOAD_REGISTRY.register(IOBasePayload, io.IOBase) +PAYLOAD_REGISTRY.register(StreamReaderPayload, StreamReader) +# try_last for giving a chance to more specialized async interables like +# multipart.BodyPartReaderPayload override the default +PAYLOAD_REGISTRY.register(AsyncIterablePayload, AsyncIterable, order=Order.try_last) diff --git a/lib/python3.1/site-packages/aiohttp/payload_streamer.py b/lib/python3.1/site-packages/aiohttp/payload_streamer.py new file mode 100644 index 0000000000000000000000000000000000000000..831fdc0a77f302acaf9a000be408fe7c9a9035aa --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/payload_streamer.py @@ -0,0 +1,78 @@ +""" +Payload implementation for coroutines as data provider. + +As a simple case, you can upload data from file:: + + @aiohttp.streamer + async def file_sender(writer, file_name=None): + with open(file_name, 'rb') as f: + chunk = f.read(2**16) + while chunk: + await writer.write(chunk) + + chunk = f.read(2**16) + +Then you can use `file_sender` like this: + + async with session.post('http://httpbin.org/post', + data=file_sender(file_name='huge_file')) as resp: + print(await resp.text()) + +..note:: Coroutine must accept `writer` as first argument + +""" + +import types +import warnings +from typing import Any, Awaitable, Callable, Dict, Tuple + +from .abc import AbstractStreamWriter +from .payload import Payload, payload_type + +__all__ = ("streamer",) + + +class _stream_wrapper: + def __init__( + self, + coro: Callable[..., Awaitable[None]], + args: Tuple[Any, ...], + kwargs: Dict[str, Any], + ) -> None: + self.coro = types.coroutine(coro) + self.args = args + self.kwargs = kwargs + + async def __call__(self, writer: AbstractStreamWriter) -> None: + await self.coro(writer, *self.args, **self.kwargs) + + +class streamer: + def __init__(self, coro: Callable[..., Awaitable[None]]) -> None: + warnings.warn( + "@streamer is deprecated, use async generators instead", + DeprecationWarning, + stacklevel=2, + ) + self.coro = coro + + def __call__(self, *args: Any, **kwargs: Any) -> _stream_wrapper: + return _stream_wrapper(self.coro, args, kwargs) + + +@payload_type(_stream_wrapper) +class StreamWrapperPayload(Payload): + async def write(self, writer: AbstractStreamWriter) -> None: + await self._value(writer) + + def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str: + raise TypeError("Unable to decode.") + + +@payload_type(streamer) +class StreamPayload(StreamWrapperPayload): + def __init__(self, value: Any, *args: Any, **kwargs: Any) -> None: + super().__init__(value(), *args, **kwargs) + + async def write(self, writer: AbstractStreamWriter) -> None: + await self._value(writer) diff --git a/lib/python3.1/site-packages/aiohttp/py.typed b/lib/python3.1/site-packages/aiohttp/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..f5642f79f21d872f010979dcf6f0c4a415acc19d --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/py.typed @@ -0,0 +1 @@ +Marker diff --git a/lib/python3.1/site-packages/aiohttp/pytest_plugin.py b/lib/python3.1/site-packages/aiohttp/pytest_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..7d59fe820d697632c9a3311ff96841ebc0ee735b --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/pytest_plugin.py @@ -0,0 +1,444 @@ +import asyncio +import contextlib +import inspect +import warnings +from typing import ( + Any, + Awaitable, + Callable, + Dict, + Iterator, + Optional, + Protocol, + Union, + overload, +) + +import pytest + +from .test_utils import ( + BaseTestServer, + RawTestServer, + TestClient, + TestServer, + loop_context, + setup_test_loop, + teardown_test_loop, + unused_port as _unused_port, +) +from .web import Application, BaseRequest, Request +from .web_protocol import _RequestHandler + +try: + import uvloop +except ImportError: # pragma: no cover + uvloop = None # type: ignore[assignment] + + +class AiohttpClient(Protocol): + @overload + async def __call__( + self, + __param: Application, + *, + server_kwargs: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> TestClient[Request, Application]: ... + @overload + async def __call__( + self, + __param: BaseTestServer, + *, + server_kwargs: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> TestClient[BaseRequest, None]: ... + + +class AiohttpServer(Protocol): + def __call__( + self, app: Application, *, port: Optional[int] = None, **kwargs: Any + ) -> Awaitable[TestServer]: ... + + +class AiohttpRawServer(Protocol): + def __call__( + self, handler: _RequestHandler, *, port: Optional[int] = None, **kwargs: Any + ) -> Awaitable[RawTestServer]: ... + + +def pytest_addoption(parser): # type: ignore[no-untyped-def] + parser.addoption( + "--aiohttp-fast", + action="store_true", + default=False, + help="run tests faster by disabling extra checks", + ) + parser.addoption( + "--aiohttp-loop", + action="store", + default="pyloop", + help="run tests with specific loop: pyloop, uvloop or all", + ) + parser.addoption( + "--aiohttp-enable-loop-debug", + action="store_true", + default=False, + help="enable event loop debug mode", + ) + + +def pytest_fixture_setup(fixturedef): # type: ignore[no-untyped-def] + """Set up pytest fixture. + + Allow fixtures to be coroutines. Run coroutine fixtures in an event loop. + """ + func = fixturedef.func + + if inspect.isasyncgenfunction(func): + # async generator fixture + is_async_gen = True + elif inspect.iscoroutinefunction(func): + # regular async fixture + is_async_gen = False + else: + # not an async fixture, nothing to do + return + + strip_request = False + if "request" not in fixturedef.argnames: + fixturedef.argnames += ("request",) + strip_request = True + + def wrapper(*args, **kwargs): # type: ignore[no-untyped-def] + request = kwargs["request"] + if strip_request: + del kwargs["request"] + + # if neither the fixture nor the test use the 'loop' fixture, + # 'getfixturevalue' will fail because the test is not parameterized + # (this can be removed someday if 'loop' is no longer parameterized) + if "loop" not in request.fixturenames: + raise Exception( + "Asynchronous fixtures must depend on the 'loop' fixture or " + "be used in tests depending from it." + ) + + _loop = request.getfixturevalue("loop") + + if is_async_gen: + # for async generators, we need to advance the generator once, + # then advance it again in a finalizer + gen = func(*args, **kwargs) + + def finalizer(): # type: ignore[no-untyped-def] + try: + return _loop.run_until_complete(gen.__anext__()) + except StopAsyncIteration: + pass + + request.addfinalizer(finalizer) + return _loop.run_until_complete(gen.__anext__()) + else: + return _loop.run_until_complete(func(*args, **kwargs)) + + fixturedef.func = wrapper + + +@pytest.fixture +def fast(request): # type: ignore[no-untyped-def] + """--fast config option""" + return request.config.getoption("--aiohttp-fast") + + +@pytest.fixture +def loop_debug(request): # type: ignore[no-untyped-def] + """--enable-loop-debug config option""" + return request.config.getoption("--aiohttp-enable-loop-debug") + + +@contextlib.contextmanager +def _runtime_warning_context(): # type: ignore[no-untyped-def] + """Context manager which checks for RuntimeWarnings. + + This exists specifically to + avoid "coroutine 'X' was never awaited" warnings being missed. + + If RuntimeWarnings occur in the context a RuntimeError is raised. + """ + with warnings.catch_warnings(record=True) as _warnings: + yield + rw = [ + "{w.filename}:{w.lineno}:{w.message}".format(w=w) + for w in _warnings + if w.category == RuntimeWarning + ] + if rw: + raise RuntimeError( + "{} Runtime Warning{},\n{}".format( + len(rw), "" if len(rw) == 1 else "s", "\n".join(rw) + ) + ) + + +@contextlib.contextmanager +def _passthrough_loop_context(loop, fast=False): # type: ignore[no-untyped-def] + """Passthrough loop context. + + Sets up and tears down a loop unless one is passed in via the loop + argument when it's passed straight through. + """ + if loop: + # loop already exists, pass it straight through + yield loop + else: + # this shadows loop_context's standard behavior + loop = setup_test_loop() + yield loop + teardown_test_loop(loop, fast=fast) + + +def pytest_pycollect_makeitem(collector, name, obj): # type: ignore[no-untyped-def] + """Fix pytest collecting for coroutines.""" + if collector.funcnamefilter(name) and inspect.iscoroutinefunction(obj): + return list(collector._genfunctions(name, obj)) + + +def pytest_pyfunc_call(pyfuncitem): # type: ignore[no-untyped-def] + """Run coroutines in an event loop instead of a normal function call.""" + fast = pyfuncitem.config.getoption("--aiohttp-fast") + if inspect.iscoroutinefunction(pyfuncitem.function): + existing_loop = ( + pyfuncitem.funcargs.get("proactor_loop") + or pyfuncitem.funcargs.get("selector_loop") + or pyfuncitem.funcargs.get("uvloop_loop") + or pyfuncitem.funcargs.get("loop", None) + ) + + with _runtime_warning_context(): + with _passthrough_loop_context(existing_loop, fast=fast) as _loop: + testargs = { + arg: pyfuncitem.funcargs[arg] + for arg in pyfuncitem._fixtureinfo.argnames + } + _loop.run_until_complete(pyfuncitem.obj(**testargs)) + + return True + + +def pytest_generate_tests(metafunc): # type: ignore[no-untyped-def] + if "loop_factory" not in metafunc.fixturenames: + return + + loops = metafunc.config.option.aiohttp_loop + avail_factories: dict[str, Callable[[], asyncio.AbstractEventLoop]] + avail_factories = {"pyloop": asyncio.new_event_loop} + + if uvloop is not None: # pragma: no cover + avail_factories["uvloop"] = uvloop.new_event_loop + + if loops == "all": + loops = "pyloop,uvloop?" + + factories = {} # type: ignore[var-annotated] + for name in loops.split(","): + required = not name.endswith("?") + name = name.strip(" ?") + if name not in avail_factories: # pragma: no cover + if required: + raise ValueError( + "Unknown loop '%s', available loops: %s" + % (name, list(factories.keys())) + ) + else: + continue + factories[name] = avail_factories[name] + metafunc.parametrize( + "loop_factory", list(factories.values()), ids=list(factories.keys()) + ) + + +@pytest.fixture +def loop( + loop_factory: Callable[[], asyncio.AbstractEventLoop], + fast: bool, + loop_debug: bool, +) -> Iterator[asyncio.AbstractEventLoop]: + """Return an instance of the event loop.""" + with loop_context(loop_factory, fast=fast) as _loop: + if loop_debug: + _loop.set_debug(True) # pragma: no cover + asyncio.set_event_loop(_loop) + yield _loop + + +@pytest.fixture +def proactor_loop() -> Iterator[asyncio.AbstractEventLoop]: + factory = asyncio.ProactorEventLoop # type: ignore[attr-defined] + + with loop_context(factory) as _loop: + asyncio.set_event_loop(_loop) + yield _loop + + +@pytest.fixture +def unused_port(aiohttp_unused_port: Callable[[], int]) -> Callable[[], int]: + warnings.warn( + "Deprecated, use aiohttp_unused_port fixture instead", + DeprecationWarning, + stacklevel=2, + ) + return aiohttp_unused_port + + +@pytest.fixture +def aiohttp_unused_port() -> Callable[[], int]: + """Return a port that is unused on the current host.""" + return _unused_port + + +@pytest.fixture +def aiohttp_server(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpServer]: + """Factory to create a TestServer instance, given an app. + + aiohttp_server(app, **kwargs) + """ + servers = [] + + async def go( + app: Application, + *, + host: str = "127.0.0.1", + port: Optional[int] = None, + **kwargs: Any, + ) -> TestServer: + server = TestServer(app, host=host, port=port) + await server.start_server(loop=loop, **kwargs) + servers.append(server) + return server + + yield go + + async def finalize() -> None: + while servers: + await servers.pop().close() + + loop.run_until_complete(finalize()) + + +@pytest.fixture +def test_server(aiohttp_server): # type: ignore[no-untyped-def] # pragma: no cover + warnings.warn( + "Deprecated, use aiohttp_server fixture instead", + DeprecationWarning, + stacklevel=2, + ) + return aiohttp_server + + +@pytest.fixture +def aiohttp_raw_server(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpRawServer]: + """Factory to create a RawTestServer instance, given a web handler. + + aiohttp_raw_server(handler, **kwargs) + """ + servers = [] + + async def go( + handler: _RequestHandler, *, port: Optional[int] = None, **kwargs: Any + ) -> RawTestServer: + server = RawTestServer(handler, port=port) + await server.start_server(loop=loop, **kwargs) + servers.append(server) + return server + + yield go + + async def finalize() -> None: + while servers: + await servers.pop().close() + + loop.run_until_complete(finalize()) + + +@pytest.fixture +def raw_test_server( # type: ignore[no-untyped-def] # pragma: no cover + aiohttp_raw_server, +): + warnings.warn( + "Deprecated, use aiohttp_raw_server fixture instead", + DeprecationWarning, + stacklevel=2, + ) + return aiohttp_raw_server + + +@pytest.fixture +def aiohttp_client(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpClient]: + """Factory to create a TestClient instance. + + aiohttp_client(app, **kwargs) + aiohttp_client(server, **kwargs) + aiohttp_client(raw_server, **kwargs) + """ + clients = [] + + @overload + async def go( + __param: Application, + *, + server_kwargs: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> TestClient[Request, Application]: ... + + @overload + async def go( + __param: BaseTestServer, + *, + server_kwargs: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> TestClient[BaseRequest, None]: ... + + async def go( + __param: Union[Application, BaseTestServer], + *args: Any, + server_kwargs: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> TestClient[Any, Any]: + if isinstance(__param, Callable) and not isinstance( # type: ignore[arg-type] + __param, (Application, BaseTestServer) + ): + __param = __param(loop, *args, **kwargs) + kwargs = {} + else: + assert not args, "args should be empty" + + if isinstance(__param, Application): + server_kwargs = server_kwargs or {} + server = TestServer(__param, loop=loop, **server_kwargs) + client = TestClient(server, loop=loop, **kwargs) + elif isinstance(__param, BaseTestServer): + client = TestClient(__param, loop=loop, **kwargs) + else: + raise ValueError("Unknown argument type: %r" % type(__param)) + + await client.start_server() + clients.append(client) + return client + + yield go + + async def finalize() -> None: + while clients: + await clients.pop().close() + + loop.run_until_complete(finalize()) + + +@pytest.fixture +def test_client(aiohttp_client): # type: ignore[no-untyped-def] # pragma: no cover + warnings.warn( + "Deprecated, use aiohttp_client fixture instead", + DeprecationWarning, + stacklevel=2, + ) + return aiohttp_client diff --git a/lib/python3.1/site-packages/aiohttp/resolver.py b/lib/python3.1/site-packages/aiohttp/resolver.py new file mode 100644 index 0000000000000000000000000000000000000000..b20e5672ce51a1aadd7768301a3a1c8eb6007bf4 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/resolver.py @@ -0,0 +1,274 @@ +import asyncio +import socket +import weakref +from typing import Any, Dict, Final, List, Optional, Tuple, Type, Union + +from .abc import AbstractResolver, ResolveResult + +__all__ = ("ThreadedResolver", "AsyncResolver", "DefaultResolver") + + +try: + import aiodns + + aiodns_default = hasattr(aiodns.DNSResolver, "getaddrinfo") +except ImportError: # pragma: no cover + aiodns = None # type: ignore[assignment] + aiodns_default = False + + +_NUMERIC_SOCKET_FLAGS = socket.AI_NUMERICHOST | socket.AI_NUMERICSERV +_NAME_SOCKET_FLAGS = socket.NI_NUMERICHOST | socket.NI_NUMERICSERV +_AI_ADDRCONFIG = socket.AI_ADDRCONFIG +if hasattr(socket, "AI_MASK"): + _AI_ADDRCONFIG &= socket.AI_MASK + + +class ThreadedResolver(AbstractResolver): + """Threaded resolver. + + Uses an Executor for synchronous getaddrinfo() calls. + concurrent.futures.ThreadPoolExecutor is used by default. + """ + + def __init__(self, loop: Optional[asyncio.AbstractEventLoop] = None) -> None: + self._loop = loop or asyncio.get_running_loop() + + async def resolve( + self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET + ) -> List[ResolveResult]: + infos = await self._loop.getaddrinfo( + host, + port, + type=socket.SOCK_STREAM, + family=family, + flags=_AI_ADDRCONFIG, + ) + + hosts: List[ResolveResult] = [] + for family, _, proto, _, address in infos: + if family == socket.AF_INET6: + if len(address) < 3: + # IPv6 is not supported by Python build, + # or IPv6 is not enabled in the host + continue + if address[3]: + # This is essential for link-local IPv6 addresses. + # LL IPv6 is a VERY rare case. Strictly speaking, we should use + # getnameinfo() unconditionally, but performance makes sense. + resolved_host, _port = await self._loop.getnameinfo( + address, _NAME_SOCKET_FLAGS + ) + port = int(_port) + else: + resolved_host, port = address[:2] + else: # IPv4 + assert family == socket.AF_INET + resolved_host, port = address # type: ignore[misc] + hosts.append( + ResolveResult( + hostname=host, + host=resolved_host, + port=port, + family=family, + proto=proto, + flags=_NUMERIC_SOCKET_FLAGS, + ) + ) + + return hosts + + async def close(self) -> None: + pass + + +class AsyncResolver(AbstractResolver): + """Use the `aiodns` package to make asynchronous DNS lookups""" + + def __init__( + self, + loop: Optional[asyncio.AbstractEventLoop] = None, + *args: Any, + **kwargs: Any, + ) -> None: + if aiodns is None: + raise RuntimeError("Resolver requires aiodns library") + + self._loop = loop or asyncio.get_running_loop() + self._manager: Optional[_DNSResolverManager] = None + # If custom args are provided, create a dedicated resolver instance + # This means each AsyncResolver with custom args gets its own + # aiodns.DNSResolver instance + if args or kwargs: + self._resolver = aiodns.DNSResolver(*args, **kwargs) + return + # Use the shared resolver from the manager for default arguments + self._manager = _DNSResolverManager() + self._resolver = self._manager.get_resolver(self, self._loop) + + if not hasattr(self._resolver, "gethostbyname"): + # aiodns 1.1 is not available, fallback to DNSResolver.query + self.resolve = self._resolve_with_query # type: ignore + + async def resolve( + self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET + ) -> List[ResolveResult]: + try: + resp = await self._resolver.getaddrinfo( + host, + port=port, + type=socket.SOCK_STREAM, + family=family, + flags=_AI_ADDRCONFIG, + ) + except aiodns.error.DNSError as exc: + msg = exc.args[1] if len(exc.args) >= 1 else "DNS lookup failed" + raise OSError(None, msg) from exc + hosts: List[ResolveResult] = [] + for node in resp.nodes: + address: Union[Tuple[bytes, int], Tuple[bytes, int, int, int]] = node.addr + family = node.family + if family == socket.AF_INET6: + if len(address) > 3 and address[3]: + # This is essential for link-local IPv6 addresses. + # LL IPv6 is a VERY rare case. Strictly speaking, we should use + # getnameinfo() unconditionally, but performance makes sense. + result = await self._resolver.getnameinfo( + (address[0].decode("ascii"), *address[1:]), + _NAME_SOCKET_FLAGS, + ) + resolved_host = result.node + else: + resolved_host = address[0].decode("ascii") + port = address[1] + else: # IPv4 + assert family == socket.AF_INET + resolved_host = address[0].decode("ascii") + port = address[1] + hosts.append( + ResolveResult( + hostname=host, + host=resolved_host, + port=port, + family=family, + proto=0, + flags=_NUMERIC_SOCKET_FLAGS, + ) + ) + + if not hosts: + raise OSError(None, "DNS lookup failed") + + return hosts + + async def _resolve_with_query( + self, host: str, port: int = 0, family: int = socket.AF_INET + ) -> List[Dict[str, Any]]: + qtype: Final = "AAAA" if family == socket.AF_INET6 else "A" + + try: + resp = await self._resolver.query(host, qtype) + except aiodns.error.DNSError as exc: + msg = exc.args[1] if len(exc.args) >= 1 else "DNS lookup failed" + raise OSError(None, msg) from exc + + hosts = [] + for rr in resp: + hosts.append( + { + "hostname": host, + "host": rr.host, + "port": port, + "family": family, + "proto": 0, + "flags": socket.AI_NUMERICHOST, + } + ) + + if not hosts: + raise OSError(None, "DNS lookup failed") + + return hosts + + async def close(self) -> None: + if self._manager: + # Release the resolver from the manager if using the shared resolver + self._manager.release_resolver(self, self._loop) + self._manager = None # Clear reference to manager + self._resolver = None # type: ignore[assignment] # Clear reference to resolver + return + # Otherwise cancel our dedicated resolver + if self._resolver is not None: + self._resolver.cancel() + self._resolver = None # type: ignore[assignment] # Clear reference + + +class _DNSResolverManager: + """Manager for aiodns.DNSResolver objects. + + This class manages shared aiodns.DNSResolver instances + with no custom arguments across different event loops. + """ + + _instance: Optional["_DNSResolverManager"] = None + + def __new__(cls) -> "_DNSResolverManager": + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._init() + return cls._instance + + def _init(self) -> None: + # Use WeakKeyDictionary to allow event loops to be garbage collected + self._loop_data: weakref.WeakKeyDictionary[ + asyncio.AbstractEventLoop, + tuple["aiodns.DNSResolver", weakref.WeakSet["AsyncResolver"]], + ] = weakref.WeakKeyDictionary() + + def get_resolver( + self, client: "AsyncResolver", loop: asyncio.AbstractEventLoop + ) -> "aiodns.DNSResolver": + """Get or create the shared aiodns.DNSResolver instance for a specific event loop. + + Args: + client: The AsyncResolver instance requesting the resolver. + This is required to track resolver usage. + loop: The event loop to use for the resolver. + """ + # Create a new resolver and client set for this loop if it doesn't exist + if loop not in self._loop_data: + resolver = aiodns.DNSResolver(loop=loop) + client_set: weakref.WeakSet["AsyncResolver"] = weakref.WeakSet() + self._loop_data[loop] = (resolver, client_set) + else: + # Get the existing resolver and client set + resolver, client_set = self._loop_data[loop] + + # Register this client with the loop + client_set.add(client) + return resolver + + def release_resolver( + self, client: "AsyncResolver", loop: asyncio.AbstractEventLoop + ) -> None: + """Release the resolver for an AsyncResolver client when it's closed. + + Args: + client: The AsyncResolver instance to release. + loop: The event loop the resolver was using. + """ + # Remove client from its loop's tracking + current_loop_data = self._loop_data.get(loop) + if current_loop_data is None: + return + resolver, client_set = current_loop_data + client_set.discard(client) + # If no more clients for this loop, cancel and remove its resolver + if not client_set: + if resolver is not None: + resolver.cancel() + del self._loop_data[loop] + + +_DefaultType = Type[Union[AsyncResolver, ThreadedResolver]] +DefaultResolver: _DefaultType = AsyncResolver if aiodns_default else ThreadedResolver diff --git a/lib/python3.1/site-packages/aiohttp/streams.py b/lib/python3.1/site-packages/aiohttp/streams.py new file mode 100644 index 0000000000000000000000000000000000000000..7a3f64d12894e9d3158ec2a8415c48c47fed0bcd --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/streams.py @@ -0,0 +1,727 @@ +import asyncio +import collections +import warnings +from typing import ( + Awaitable, + Callable, + Deque, + Final, + Generic, + List, + Optional, + Tuple, + TypeVar, +) + +from .base_protocol import BaseProtocol +from .helpers import ( + _EXC_SENTINEL, + BaseTimerContext, + TimerNoop, + set_exception, + set_result, +) +from .log import internal_logger + +__all__ = ( + "EMPTY_PAYLOAD", + "EofStream", + "StreamReader", + "DataQueue", +) + +_T = TypeVar("_T") + + +class EofStream(Exception): + """eof stream indication.""" + + +class AsyncStreamIterator(Generic[_T]): + + __slots__ = ("read_func",) + + def __init__(self, read_func: Callable[[], Awaitable[_T]]) -> None: + self.read_func = read_func + + def __aiter__(self) -> "AsyncStreamIterator[_T]": + return self + + async def __anext__(self) -> _T: + try: + rv = await self.read_func() + except EofStream: + raise StopAsyncIteration + if rv == b"": + raise StopAsyncIteration + return rv + + +class ChunkTupleAsyncStreamIterator: + + __slots__ = ("_stream",) + + def __init__(self, stream: "StreamReader") -> None: + self._stream = stream + + def __aiter__(self) -> "ChunkTupleAsyncStreamIterator": + return self + + async def __anext__(self) -> Tuple[bytes, bool]: + rv = await self._stream.readchunk() + if rv == (b"", False): + raise StopAsyncIteration + return rv + + +class AsyncStreamReaderMixin: + + __slots__ = () + + def __aiter__(self) -> AsyncStreamIterator[bytes]: + return AsyncStreamIterator(self.readline) # type: ignore[attr-defined] + + def iter_chunked(self, n: int) -> AsyncStreamIterator[bytes]: + """Returns an asynchronous iterator that yields chunks of size n.""" + return AsyncStreamIterator(lambda: self.read(n)) # type: ignore[attr-defined] + + def iter_any(self) -> AsyncStreamIterator[bytes]: + """Yield all available data as soon as it is received.""" + return AsyncStreamIterator(self.readany) # type: ignore[attr-defined] + + def iter_chunks(self) -> ChunkTupleAsyncStreamIterator: + """Yield chunks of data as they are received by the server. + + The yielded objects are tuples + of (bytes, bool) as returned by the StreamReader.readchunk method. + """ + return ChunkTupleAsyncStreamIterator(self) # type: ignore[arg-type] + + +class StreamReader(AsyncStreamReaderMixin): + """An enhancement of asyncio.StreamReader. + + Supports asynchronous iteration by line, chunk or as available:: + + async for line in reader: + ... + async for chunk in reader.iter_chunked(1024): + ... + async for slice in reader.iter_any(): + ... + + """ + + __slots__ = ( + "_protocol", + "_low_water", + "_high_water", + "_loop", + "_size", + "_cursor", + "_http_chunk_splits", + "_buffer", + "_buffer_offset", + "_eof", + "_waiter", + "_eof_waiter", + "_exception", + "_timer", + "_eof_callbacks", + "_eof_counter", + "total_bytes", + ) + + def __init__( + self, + protocol: BaseProtocol, + limit: int, + *, + timer: Optional[BaseTimerContext] = None, + loop: Optional[asyncio.AbstractEventLoop] = None, + ) -> None: + self._protocol = protocol + self._low_water = limit + self._high_water = limit * 2 + if loop is None: + loop = asyncio.get_event_loop() + self._loop = loop + self._size = 0 + self._cursor = 0 + self._http_chunk_splits: Optional[List[int]] = None + self._buffer: Deque[bytes] = collections.deque() + self._buffer_offset = 0 + self._eof = False + self._waiter: Optional[asyncio.Future[None]] = None + self._eof_waiter: Optional[asyncio.Future[None]] = None + self._exception: Optional[BaseException] = None + self._timer = TimerNoop() if timer is None else timer + self._eof_callbacks: List[Callable[[], None]] = [] + self._eof_counter = 0 + self.total_bytes = 0 + + def __repr__(self) -> str: + info = [self.__class__.__name__] + if self._size: + info.append("%d bytes" % self._size) + if self._eof: + info.append("eof") + if self._low_water != 2**16: # default limit + info.append("low=%d high=%d" % (self._low_water, self._high_water)) + if self._waiter: + info.append("w=%r" % self._waiter) + if self._exception: + info.append("e=%r" % self._exception) + return "<%s>" % " ".join(info) + + def get_read_buffer_limits(self) -> Tuple[int, int]: + return (self._low_water, self._high_water) + + def exception(self) -> Optional[BaseException]: + return self._exception + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + self._exception = exc + self._eof_callbacks.clear() + + waiter = self._waiter + if waiter is not None: + self._waiter = None + set_exception(waiter, exc, exc_cause) + + waiter = self._eof_waiter + if waiter is not None: + self._eof_waiter = None + set_exception(waiter, exc, exc_cause) + + def on_eof(self, callback: Callable[[], None]) -> None: + if self._eof: + try: + callback() + except Exception: + internal_logger.exception("Exception in eof callback") + else: + self._eof_callbacks.append(callback) + + def feed_eof(self) -> None: + self._eof = True + + waiter = self._waiter + if waiter is not None: + self._waiter = None + set_result(waiter, None) + + waiter = self._eof_waiter + if waiter is not None: + self._eof_waiter = None + set_result(waiter, None) + + if self._protocol._reading_paused: + self._protocol.resume_reading() + + for cb in self._eof_callbacks: + try: + cb() + except Exception: + internal_logger.exception("Exception in eof callback") + + self._eof_callbacks.clear() + + def is_eof(self) -> bool: + """Return True if 'feed_eof' was called.""" + return self._eof + + def at_eof(self) -> bool: + """Return True if the buffer is empty and 'feed_eof' was called.""" + return self._eof and not self._buffer + + async def wait_eof(self) -> None: + if self._eof: + return + + assert self._eof_waiter is None + self._eof_waiter = self._loop.create_future() + try: + await self._eof_waiter + finally: + self._eof_waiter = None + + def unread_data(self, data: bytes) -> None: + """rollback reading some data from stream, inserting it to buffer head.""" + warnings.warn( + "unread_data() is deprecated " + "and will be removed in future releases (#3260)", + DeprecationWarning, + stacklevel=2, + ) + if not data: + return + + if self._buffer_offset: + self._buffer[0] = self._buffer[0][self._buffer_offset :] + self._buffer_offset = 0 + self._size += len(data) + self._cursor -= len(data) + self._buffer.appendleft(data) + self._eof_counter = 0 + + # TODO: size is ignored, remove the param later + def feed_data(self, data: bytes, size: int = 0) -> None: + assert not self._eof, "feed_data after feed_eof" + + if not data: + return + + data_len = len(data) + self._size += data_len + self._buffer.append(data) + self.total_bytes += data_len + + waiter = self._waiter + if waiter is not None: + self._waiter = None + set_result(waiter, None) + + if self._size > self._high_water and not self._protocol._reading_paused: + self._protocol.pause_reading() + + def begin_http_chunk_receiving(self) -> None: + if self._http_chunk_splits is None: + if self.total_bytes: + raise RuntimeError( + "Called begin_http_chunk_receiving when some data was already fed" + ) + self._http_chunk_splits = [] + + def end_http_chunk_receiving(self) -> None: + if self._http_chunk_splits is None: + raise RuntimeError( + "Called end_chunk_receiving without calling " + "begin_chunk_receiving first" + ) + + # self._http_chunk_splits contains logical byte offsets from start of + # the body transfer. Each offset is the offset of the end of a chunk. + # "Logical" means bytes, accessible for a user. + # If no chunks containing logical data were received, current position + # is difinitely zero. + pos = self._http_chunk_splits[-1] if self._http_chunk_splits else 0 + + if self.total_bytes == pos: + # We should not add empty chunks here. So we check for that. + # Note, when chunked + gzip is used, we can receive a chunk + # of compressed data, but that data may not be enough for gzip FSM + # to yield any uncompressed data. That's why current position may + # not change after receiving a chunk. + return + + self._http_chunk_splits.append(self.total_bytes) + + # wake up readchunk when end of http chunk received + waiter = self._waiter + if waiter is not None: + self._waiter = None + set_result(waiter, None) + + async def _wait(self, func_name: str) -> None: + if not self._protocol.connected: + raise RuntimeError("Connection closed.") + + # StreamReader uses a future to link the protocol feed_data() method + # to a read coroutine. Running two read coroutines at the same time + # would have an unexpected behaviour. It would not possible to know + # which coroutine would get the next data. + if self._waiter is not None: + raise RuntimeError( + "%s() called while another coroutine is " + "already waiting for incoming data" % func_name + ) + + waiter = self._waiter = self._loop.create_future() + try: + with self._timer: + await waiter + finally: + self._waiter = None + + async def readline(self) -> bytes: + return await self.readuntil() + + async def readuntil(self, separator: bytes = b"\n") -> bytes: + seplen = len(separator) + if seplen == 0: + raise ValueError("Separator should be at least one-byte string") + + if self._exception is not None: + raise self._exception + + chunk = b"" + chunk_size = 0 + not_enough = True + + while not_enough: + while self._buffer and not_enough: + offset = self._buffer_offset + ichar = self._buffer[0].find(separator, offset) + 1 + # Read from current offset to found separator or to the end. + data = self._read_nowait_chunk( + ichar - offset + seplen - 1 if ichar else -1 + ) + chunk += data + chunk_size += len(data) + if ichar: + not_enough = False + + if chunk_size > self._high_water: + raise ValueError("Chunk too big") + + if self._eof: + break + + if not_enough: + await self._wait("readuntil") + + return chunk + + async def read(self, n: int = -1) -> bytes: + if self._exception is not None: + raise self._exception + + # migration problem; with DataQueue you have to catch + # EofStream exception, so common way is to run payload.read() inside + # infinite loop. what can cause real infinite loop with StreamReader + # lets keep this code one major release. + if __debug__: + if self._eof and not self._buffer: + self._eof_counter = getattr(self, "_eof_counter", 0) + 1 + if self._eof_counter > 5: + internal_logger.warning( + "Multiple access to StreamReader in eof state, " + "might be infinite loop.", + stack_info=True, + ) + + if not n: + return b"" + + if n < 0: + # This used to just loop creating a new waiter hoping to + # collect everything in self._buffer, but that would + # deadlock if the subprocess sends more than self.limit + # bytes. So just call self.readany() until EOF. + blocks = [] + while True: + block = await self.readany() + if not block: + break + blocks.append(block) + return b"".join(blocks) + + # TODO: should be `if` instead of `while` + # because waiter maybe triggered on chunk end, + # without feeding any data + while not self._buffer and not self._eof: + await self._wait("read") + + return self._read_nowait(n) + + async def readany(self) -> bytes: + if self._exception is not None: + raise self._exception + + # TODO: should be `if` instead of `while` + # because waiter maybe triggered on chunk end, + # without feeding any data + while not self._buffer and not self._eof: + await self._wait("readany") + + return self._read_nowait(-1) + + async def readchunk(self) -> Tuple[bytes, bool]: + """Returns a tuple of (data, end_of_http_chunk). + + When chunked transfer + encoding is used, end_of_http_chunk is a boolean indicating if the end + of the data corresponds to the end of a HTTP chunk , otherwise it is + always False. + """ + while True: + if self._exception is not None: + raise self._exception + + while self._http_chunk_splits: + pos = self._http_chunk_splits.pop(0) + if pos == self._cursor: + return (b"", True) + if pos > self._cursor: + return (self._read_nowait(pos - self._cursor), True) + internal_logger.warning( + "Skipping HTTP chunk end due to data " + "consumption beyond chunk boundary" + ) + + if self._buffer: + return (self._read_nowait_chunk(-1), False) + # return (self._read_nowait(-1), False) + + if self._eof: + # Special case for signifying EOF. + # (b'', True) is not a final return value actually. + return (b"", False) + + await self._wait("readchunk") + + async def readexactly(self, n: int) -> bytes: + if self._exception is not None: + raise self._exception + + blocks: List[bytes] = [] + while n > 0: + block = await self.read(n) + if not block: + partial = b"".join(blocks) + raise asyncio.IncompleteReadError(partial, len(partial) + n) + blocks.append(block) + n -= len(block) + + return b"".join(blocks) + + def read_nowait(self, n: int = -1) -> bytes: + # default was changed to be consistent with .read(-1) + # + # I believe the most users don't know about the method and + # they are not affected. + if self._exception is not None: + raise self._exception + + if self._waiter and not self._waiter.done(): + raise RuntimeError( + "Called while some coroutine is waiting for incoming data." + ) + + return self._read_nowait(n) + + def _read_nowait_chunk(self, n: int) -> bytes: + first_buffer = self._buffer[0] + offset = self._buffer_offset + if n != -1 and len(first_buffer) - offset > n: + data = first_buffer[offset : offset + n] + self._buffer_offset += n + + elif offset: + self._buffer.popleft() + data = first_buffer[offset:] + self._buffer_offset = 0 + + else: + data = self._buffer.popleft() + + data_len = len(data) + self._size -= data_len + self._cursor += data_len + + chunk_splits = self._http_chunk_splits + # Prevent memory leak: drop useless chunk splits + while chunk_splits and chunk_splits[0] < self._cursor: + chunk_splits.pop(0) + + if self._size < self._low_water and self._protocol._reading_paused: + self._protocol.resume_reading() + return data + + def _read_nowait(self, n: int) -> bytes: + """Read not more than n bytes, or whole buffer if n == -1""" + self._timer.assert_timeout() + + chunks = [] + while self._buffer: + chunk = self._read_nowait_chunk(n) + chunks.append(chunk) + if n != -1: + n -= len(chunk) + if n == 0: + break + + return b"".join(chunks) if chunks else b"" + + +class EmptyStreamReader(StreamReader): # lgtm [py/missing-call-to-init] + + __slots__ = ("_read_eof_chunk",) + + def __init__(self) -> None: + self._read_eof_chunk = False + self.total_bytes = 0 + + def __repr__(self) -> str: + return "<%s>" % self.__class__.__name__ + + def exception(self) -> Optional[BaseException]: + return None + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + pass + + def on_eof(self, callback: Callable[[], None]) -> None: + try: + callback() + except Exception: + internal_logger.exception("Exception in eof callback") + + def feed_eof(self) -> None: + pass + + def is_eof(self) -> bool: + return True + + def at_eof(self) -> bool: + return True + + async def wait_eof(self) -> None: + return + + def feed_data(self, data: bytes, n: int = 0) -> None: + pass + + async def readline(self) -> bytes: + return b"" + + async def read(self, n: int = -1) -> bytes: + return b"" + + # TODO add async def readuntil + + async def readany(self) -> bytes: + return b"" + + async def readchunk(self) -> Tuple[bytes, bool]: + if not self._read_eof_chunk: + self._read_eof_chunk = True + return (b"", False) + + return (b"", True) + + async def readexactly(self, n: int) -> bytes: + raise asyncio.IncompleteReadError(b"", n) + + def read_nowait(self, n: int = -1) -> bytes: + return b"" + + +EMPTY_PAYLOAD: Final[StreamReader] = EmptyStreamReader() + + +class DataQueue(Generic[_T]): + """DataQueue is a general-purpose blocking queue with one reader.""" + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + self._eof = False + self._waiter: Optional[asyncio.Future[None]] = None + self._exception: Optional[BaseException] = None + self._buffer: Deque[Tuple[_T, int]] = collections.deque() + + def __len__(self) -> int: + return len(self._buffer) + + def is_eof(self) -> bool: + return self._eof + + def at_eof(self) -> bool: + return self._eof and not self._buffer + + def exception(self) -> Optional[BaseException]: + return self._exception + + def set_exception( + self, + exc: BaseException, + exc_cause: BaseException = _EXC_SENTINEL, + ) -> None: + self._eof = True + self._exception = exc + if (waiter := self._waiter) is not None: + self._waiter = None + set_exception(waiter, exc, exc_cause) + + def feed_data(self, data: _T, size: int = 0) -> None: + self._buffer.append((data, size)) + if (waiter := self._waiter) is not None: + self._waiter = None + set_result(waiter, None) + + def feed_eof(self) -> None: + self._eof = True + if (waiter := self._waiter) is not None: + self._waiter = None + set_result(waiter, None) + + async def read(self) -> _T: + if not self._buffer and not self._eof: + assert not self._waiter + self._waiter = self._loop.create_future() + try: + await self._waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._waiter = None + raise + if self._buffer: + data, _ = self._buffer.popleft() + return data + if self._exception is not None: + raise self._exception + raise EofStream + + def __aiter__(self) -> AsyncStreamIterator[_T]: + return AsyncStreamIterator(self.read) + + +class FlowControlDataQueue(DataQueue[_T]): + """FlowControlDataQueue resumes and pauses an underlying stream. + + It is a destination for parsed data. + + This class is deprecated and will be removed in version 4.0. + """ + + def __init__( + self, protocol: BaseProtocol, limit: int, *, loop: asyncio.AbstractEventLoop + ) -> None: + super().__init__(loop=loop) + self._size = 0 + self._protocol = protocol + self._limit = limit * 2 + + def feed_data(self, data: _T, size: int = 0) -> None: + super().feed_data(data, size) + self._size += size + + if self._size > self._limit and not self._protocol._reading_paused: + self._protocol.pause_reading() + + async def read(self) -> _T: + if not self._buffer and not self._eof: + assert not self._waiter + self._waiter = self._loop.create_future() + try: + await self._waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._waiter = None + raise + if self._buffer: + data, size = self._buffer.popleft() + self._size -= size + if self._size < self._limit and self._protocol._reading_paused: + self._protocol.resume_reading() + return data + if self._exception is not None: + raise self._exception + raise EofStream diff --git a/lib/python3.1/site-packages/aiohttp/tcp_helpers.py b/lib/python3.1/site-packages/aiohttp/tcp_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..88b244223741ad2decb6cb612eae644fae88b2b2 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/tcp_helpers.py @@ -0,0 +1,37 @@ +"""Helper methods to tune a TCP connection""" + +import asyncio +import socket +from contextlib import suppress +from typing import Optional # noqa + +__all__ = ("tcp_keepalive", "tcp_nodelay") + + +if hasattr(socket, "SO_KEEPALIVE"): + + def tcp_keepalive(transport: asyncio.Transport) -> None: + sock = transport.get_extra_info("socket") + if sock is not None: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + +else: + + def tcp_keepalive(transport: asyncio.Transport) -> None: # pragma: no cover + pass + + +def tcp_nodelay(transport: asyncio.Transport, value: bool) -> None: + sock = transport.get_extra_info("socket") + + if sock is None: + return + + if sock.family not in (socket.AF_INET, socket.AF_INET6): + return + + value = bool(value) + + # socket may be closed already, on windows OSError get raised + with suppress(OSError): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, value) diff --git a/lib/python3.1/site-packages/aiohttp/test_utils.py b/lib/python3.1/site-packages/aiohttp/test_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..87c31427867f90244c11c8440a5e1f47fc5a079f --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/test_utils.py @@ -0,0 +1,774 @@ +"""Utilities shared by tests.""" + +import asyncio +import contextlib +import gc +import inspect +import ipaddress +import os +import socket +import sys +import warnings +from abc import ABC, abstractmethod +from types import TracebackType +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Generic, + Iterator, + List, + Optional, + Type, + TypeVar, + cast, + overload, +) +from unittest import IsolatedAsyncioTestCase, mock + +from aiosignal import Signal +from multidict import CIMultiDict, CIMultiDictProxy +from yarl import URL + +import aiohttp +from aiohttp.client import ( + _RequestContextManager, + _RequestOptions, + _WSRequestContextManager, +) + +from . import ClientSession, hdrs +from .abc import AbstractCookieJar +from .client_reqrep import ClientResponse +from .client_ws import ClientWebSocketResponse +from .helpers import sentinel +from .http import HttpVersion, RawRequestMessage +from .streams import EMPTY_PAYLOAD, StreamReader +from .typedefs import StrOrURL +from .web import ( + Application, + AppRunner, + BaseRequest, + BaseRunner, + Request, + Server, + ServerRunner, + SockSite, + UrlMappingMatchInfo, +) +from .web_protocol import _RequestHandler + +if TYPE_CHECKING: + from ssl import SSLContext +else: + SSLContext = None + +if sys.version_info >= (3, 11) and TYPE_CHECKING: + from typing import Unpack + +if sys.version_info >= (3, 11): + from typing import Self +else: + Self = Any + +_ApplicationNone = TypeVar("_ApplicationNone", Application, None) +_Request = TypeVar("_Request", bound=BaseRequest) + +REUSE_ADDRESS = os.name == "posix" and sys.platform != "cygwin" + + +def get_unused_port_socket( + host: str, family: socket.AddressFamily = socket.AF_INET +) -> socket.socket: + return get_port_socket(host, 0, family) + + +def get_port_socket( + host: str, port: int, family: socket.AddressFamily +) -> socket.socket: + s = socket.socket(family, socket.SOCK_STREAM) + if REUSE_ADDRESS: + # Windows has different semantics for SO_REUSEADDR, + # so don't set it. Ref: + # https://docs.microsoft.com/en-us/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((host, port)) + return s + + +def unused_port() -> int: + """Return a port that is unused on the current host.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return cast(int, s.getsockname()[1]) + + +class BaseTestServer(ABC): + __test__ = False + + def __init__( + self, + *, + scheme: str = "", + loop: Optional[asyncio.AbstractEventLoop] = None, + host: str = "127.0.0.1", + port: Optional[int] = None, + skip_url_asserts: bool = False, + socket_factory: Callable[ + [str, int, socket.AddressFamily], socket.socket + ] = get_port_socket, + **kwargs: Any, + ) -> None: + self._loop = loop + self.runner: Optional[BaseRunner] = None + self._root: Optional[URL] = None + self.host = host + self.port = port + self._closed = False + self.scheme = scheme + self.skip_url_asserts = skip_url_asserts + self.socket_factory = socket_factory + + async def start_server( + self, loop: Optional[asyncio.AbstractEventLoop] = None, **kwargs: Any + ) -> None: + if self.runner: + return + self._loop = loop + self._ssl = kwargs.pop("ssl", None) + self.runner = await self._make_runner(handler_cancellation=True, **kwargs) + await self.runner.setup() + if not self.port: + self.port = 0 + absolute_host = self.host + try: + version = ipaddress.ip_address(self.host).version + except ValueError: + version = 4 + if version == 6: + absolute_host = f"[{self.host}]" + family = socket.AF_INET6 if version == 6 else socket.AF_INET + _sock = self.socket_factory(self.host, self.port, family) + self.host, self.port = _sock.getsockname()[:2] + site = SockSite(self.runner, sock=_sock, ssl_context=self._ssl) + await site.start() + server = site._server + assert server is not None + sockets = server.sockets # type: ignore[attr-defined] + assert sockets is not None + self.port = sockets[0].getsockname()[1] + if not self.scheme: + self.scheme = "https" if self._ssl else "http" + self._root = URL(f"{self.scheme}://{absolute_host}:{self.port}") + + @abstractmethod # pragma: no cover + async def _make_runner(self, **kwargs: Any) -> BaseRunner: + pass + + def make_url(self, path: StrOrURL) -> URL: + assert self._root is not None + url = URL(path) + if not self.skip_url_asserts: + assert not url.absolute + return self._root.join(url) + else: + return URL(str(self._root) + str(path)) + + @property + def started(self) -> bool: + return self.runner is not None + + @property + def closed(self) -> bool: + return self._closed + + @property + def handler(self) -> Server: + # for backward compatibility + # web.Server instance + runner = self.runner + assert runner is not None + assert runner.server is not None + return runner.server + + async def close(self) -> None: + """Close all fixtures created by the test client. + + After that point, the TestClient is no longer usable. + + This is an idempotent function: running close multiple times + will not have any additional effects. + + close is also run when the object is garbage collected, and on + exit when used as a context manager. + + """ + if self.started and not self.closed: + assert self.runner is not None + await self.runner.cleanup() + self._root = None + self.port = None + self._closed = True + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: + # __exit__ should exist in pair with __enter__ but never executed + pass # pragma: no cover + + async def __aenter__(self) -> "BaseTestServer": + await self.start_server(loop=self._loop) + return self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: + await self.close() + + +class TestServer(BaseTestServer): + def __init__( + self, + app: Application, + *, + scheme: str = "", + host: str = "127.0.0.1", + port: Optional[int] = None, + **kwargs: Any, + ): + self.app = app + super().__init__(scheme=scheme, host=host, port=port, **kwargs) + + async def _make_runner(self, **kwargs: Any) -> BaseRunner: + return AppRunner(self.app, **kwargs) + + +class RawTestServer(BaseTestServer): + def __init__( + self, + handler: _RequestHandler, + *, + scheme: str = "", + host: str = "127.0.0.1", + port: Optional[int] = None, + **kwargs: Any, + ) -> None: + self._handler = handler + super().__init__(scheme=scheme, host=host, port=port, **kwargs) + + async def _make_runner(self, debug: bool = True, **kwargs: Any) -> ServerRunner: + srv = Server(self._handler, loop=self._loop, debug=debug, **kwargs) + return ServerRunner(srv, debug=debug, **kwargs) + + +class TestClient(Generic[_Request, _ApplicationNone]): + """ + A test client implementation. + + To write functional tests for aiohttp based servers. + + """ + + __test__ = False + + @overload + def __init__( + self: "TestClient[Request, Application]", + server: TestServer, + *, + cookie_jar: Optional[AbstractCookieJar] = None, + **kwargs: Any, + ) -> None: ... + @overload + def __init__( + self: "TestClient[_Request, None]", + server: BaseTestServer, + *, + cookie_jar: Optional[AbstractCookieJar] = None, + **kwargs: Any, + ) -> None: ... + def __init__( + self, + server: BaseTestServer, + *, + cookie_jar: Optional[AbstractCookieJar] = None, + loop: Optional[asyncio.AbstractEventLoop] = None, + **kwargs: Any, + ) -> None: + if not isinstance(server, BaseTestServer): + raise TypeError( + "server must be TestServer instance, found type: %r" % type(server) + ) + self._server = server + self._loop = loop + if cookie_jar is None: + cookie_jar = aiohttp.CookieJar(unsafe=True, loop=loop) + self._session = ClientSession(loop=loop, cookie_jar=cookie_jar, **kwargs) + self._session._retry_connection = False + self._closed = False + self._responses: List[ClientResponse] = [] + self._websockets: List[ClientWebSocketResponse] = [] + + async def start_server(self) -> None: + await self._server.start_server(loop=self._loop) + + @property + def host(self) -> str: + return self._server.host + + @property + def port(self) -> Optional[int]: + return self._server.port + + @property + def server(self) -> BaseTestServer: + return self._server + + @property + def app(self) -> _ApplicationNone: + return getattr(self._server, "app", None) # type: ignore[return-value] + + @property + def session(self) -> ClientSession: + """An internal aiohttp.ClientSession. + + Unlike the methods on the TestClient, client session requests + do not automatically include the host in the url queried, and + will require an absolute path to the resource. + + """ + return self._session + + def make_url(self, path: StrOrURL) -> URL: + return self._server.make_url(path) + + async def _request( + self, method: str, path: StrOrURL, **kwargs: Any + ) -> ClientResponse: + resp = await self._session.request(method, self.make_url(path), **kwargs) + # save it to close later + self._responses.append(resp) + return resp + + if sys.version_info >= (3, 11) and TYPE_CHECKING: + + def request( + self, method: str, path: StrOrURL, **kwargs: Unpack[_RequestOptions] + ) -> _RequestContextManager: ... + + def get( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def options( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def head( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def post( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def put( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def patch( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + def delete( + self, + path: StrOrURL, + **kwargs: Unpack[_RequestOptions], + ) -> _RequestContextManager: ... + + else: + + def request( + self, method: str, path: StrOrURL, **kwargs: Any + ) -> _RequestContextManager: + """Routes a request to tested http server. + + The interface is identical to aiohttp.ClientSession.request, + except the loop kwarg is overridden by the instance used by the + test server. + + """ + return _RequestContextManager(self._request(method, path, **kwargs)) + + def get(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP GET request.""" + return _RequestContextManager(self._request(hdrs.METH_GET, path, **kwargs)) + + def post(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP POST request.""" + return _RequestContextManager(self._request(hdrs.METH_POST, path, **kwargs)) + + def options(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP OPTIONS request.""" + return _RequestContextManager( + self._request(hdrs.METH_OPTIONS, path, **kwargs) + ) + + def head(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP HEAD request.""" + return _RequestContextManager(self._request(hdrs.METH_HEAD, path, **kwargs)) + + def put(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP PUT request.""" + return _RequestContextManager(self._request(hdrs.METH_PUT, path, **kwargs)) + + def patch(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP PATCH request.""" + return _RequestContextManager( + self._request(hdrs.METH_PATCH, path, **kwargs) + ) + + def delete(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager: + """Perform an HTTP PATCH request.""" + return _RequestContextManager( + self._request(hdrs.METH_DELETE, path, **kwargs) + ) + + def ws_connect(self, path: StrOrURL, **kwargs: Any) -> _WSRequestContextManager: + """Initiate websocket connection. + + The api corresponds to aiohttp.ClientSession.ws_connect. + + """ + return _WSRequestContextManager(self._ws_connect(path, **kwargs)) + + async def _ws_connect( + self, path: StrOrURL, **kwargs: Any + ) -> ClientWebSocketResponse: + ws = await self._session.ws_connect(self.make_url(path), **kwargs) + self._websockets.append(ws) + return ws + + async def close(self) -> None: + """Close all fixtures created by the test client. + + After that point, the TestClient is no longer usable. + + This is an idempotent function: running close multiple times + will not have any additional effects. + + close is also run on exit when used as a(n) (asynchronous) + context manager. + + """ + if not self._closed: + for resp in self._responses: + resp.close() + for ws in self._websockets: + await ws.close() + await self._session.close() + await self._server.close() + self._closed = True + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc: Optional[BaseException], + tb: Optional[TracebackType], + ) -> None: + # __exit__ should exist in pair with __enter__ but never executed + pass # pragma: no cover + + async def __aenter__(self) -> Self: + await self.start_server() + return self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc: Optional[BaseException], + tb: Optional[TracebackType], + ) -> None: + await self.close() + + +class AioHTTPTestCase(IsolatedAsyncioTestCase): + """A base class to allow for unittest web applications using aiohttp. + + Provides the following: + + * self.client (aiohttp.test_utils.TestClient): an aiohttp test client. + * self.loop (asyncio.BaseEventLoop): the event loop in which the + application and server are running. + * self.app (aiohttp.web.Application): the application returned by + self.get_application() + + Note that the TestClient's methods are asynchronous: you have to + execute function on the test client using asynchronous methods. + """ + + async def get_application(self) -> Application: + """Get application. + + This method should be overridden + to return the aiohttp.web.Application + object to test. + """ + return self.get_app() + + def get_app(self) -> Application: + """Obsolete method used to constructing web application. + + Use .get_application() coroutine instead. + """ + raise RuntimeError("Did you forget to define get_application()?") + + async def asyncSetUp(self) -> None: + self.loop = asyncio.get_running_loop() + return await self.setUpAsync() + + async def setUpAsync(self) -> None: + self.app = await self.get_application() + self.server = await self.get_server(self.app) + self.client = await self.get_client(self.server) + + await self.client.start_server() + + async def asyncTearDown(self) -> None: + return await self.tearDownAsync() + + async def tearDownAsync(self) -> None: + await self.client.close() + + async def get_server(self, app: Application) -> TestServer: + """Return a TestServer instance.""" + return TestServer(app, loop=self.loop) + + async def get_client(self, server: TestServer) -> TestClient[Request, Application]: + """Return a TestClient instance.""" + return TestClient(server, loop=self.loop) + + +def unittest_run_loop(func: Any, *args: Any, **kwargs: Any) -> Any: + """ + A decorator dedicated to use with asynchronous AioHTTPTestCase test methods. + + In 3.8+, this does nothing. + """ + warnings.warn( + "Decorator `@unittest_run_loop` is no longer needed in aiohttp 3.8+", + DeprecationWarning, + stacklevel=2, + ) + return func + + +_LOOP_FACTORY = Callable[[], asyncio.AbstractEventLoop] + + +@contextlib.contextmanager +def loop_context( + loop_factory: _LOOP_FACTORY = asyncio.new_event_loop, fast: bool = False +) -> Iterator[asyncio.AbstractEventLoop]: + """A contextmanager that creates an event_loop, for test purposes. + + Handles the creation and cleanup of a test loop. + """ + loop = setup_test_loop(loop_factory) + yield loop + teardown_test_loop(loop, fast=fast) + + +def setup_test_loop( + loop_factory: _LOOP_FACTORY = asyncio.new_event_loop, +) -> asyncio.AbstractEventLoop: + """Create and return an asyncio.BaseEventLoop instance. + + The caller should also call teardown_test_loop, + once they are done with the loop. + """ + loop = loop_factory() + asyncio.set_event_loop(loop) + return loop + + +def teardown_test_loop(loop: asyncio.AbstractEventLoop, fast: bool = False) -> None: + """Teardown and cleanup an event_loop created by setup_test_loop.""" + closed = loop.is_closed() + if not closed: + loop.call_soon(loop.stop) + loop.run_forever() + loop.close() + + if not fast: + gc.collect() + + asyncio.set_event_loop(None) + + +def _create_app_mock() -> mock.MagicMock: + def get_dict(app: Any, key: str) -> Any: + return app.__app_dict[key] + + def set_dict(app: Any, key: str, value: Any) -> None: + app.__app_dict[key] = value + + app = mock.MagicMock(spec=Application) + app.__app_dict = {} + app.__getitem__ = get_dict + app.__setitem__ = set_dict + + app._debug = False + app.on_response_prepare = Signal(app) + app.on_response_prepare.freeze() + return app + + +def _create_transport(sslcontext: Optional[SSLContext] = None) -> mock.Mock: + transport = mock.Mock() + + def get_extra_info(key: str) -> Optional[SSLContext]: + if key == "sslcontext": + return sslcontext + else: + return None + + transport.get_extra_info.side_effect = get_extra_info + return transport + + +def make_mocked_request( + method: str, + path: str, + headers: Any = None, + *, + match_info: Any = sentinel, + version: HttpVersion = HttpVersion(1, 1), + closing: bool = False, + app: Any = None, + writer: Any = sentinel, + protocol: Any = sentinel, + transport: Any = sentinel, + payload: StreamReader = EMPTY_PAYLOAD, + sslcontext: Optional[SSLContext] = None, + client_max_size: int = 1024**2, + loop: Any = ..., +) -> Request: + """Creates mocked web.Request testing purposes. + + Useful in unit tests, when spinning full web server is overkill or + specific conditions and errors are hard to trigger. + """ + task = mock.Mock() + if loop is ...: + # no loop passed, try to get the current one if + # its is running as we need a real loop to create + # executor jobs to be able to do testing + # with a real executor + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = mock.Mock() + loop.create_future.return_value = () + + if version < HttpVersion(1, 1): + closing = True + + if headers: + headers = CIMultiDictProxy(CIMultiDict(headers)) + raw_hdrs = tuple( + (k.encode("utf-8"), v.encode("utf-8")) for k, v in headers.items() + ) + else: + headers = CIMultiDictProxy(CIMultiDict()) + raw_hdrs = () + + chunked = "chunked" in headers.get(hdrs.TRANSFER_ENCODING, "").lower() + + message = RawRequestMessage( + method, + path, + version, + headers, + raw_hdrs, + closing, + None, + False, + chunked, + URL(path), + ) + if app is None: + app = _create_app_mock() + + if transport is sentinel: + transport = _create_transport(sslcontext) + + if protocol is sentinel: + protocol = mock.Mock() + protocol.transport = transport + type(protocol).peername = mock.PropertyMock( + return_value=transport.get_extra_info("peername") + ) + type(protocol).ssl_context = mock.PropertyMock(return_value=sslcontext) + + if writer is sentinel: + writer = mock.Mock() + writer.write_headers = make_mocked_coro(None) + writer.write = make_mocked_coro(None) + writer.write_eof = make_mocked_coro(None) + writer.drain = make_mocked_coro(None) + writer.transport = transport + + protocol.transport = transport + protocol.writer = writer + + req = Request( + message, payload, protocol, writer, task, loop, client_max_size=client_max_size + ) + + match_info = UrlMappingMatchInfo( + {} if match_info is sentinel else match_info, mock.Mock() + ) + match_info.add_app(app) + req._match_info = match_info + + return req + + +def make_mocked_coro( + return_value: Any = sentinel, raise_exception: Any = sentinel +) -> Any: + """Creates a coroutine mock.""" + + async def mock_coro(*args: Any, **kwargs: Any) -> Any: + if raise_exception is not sentinel: + raise raise_exception + if not inspect.isawaitable(return_value): + return return_value + await return_value + + return mock.Mock(wraps=mock_coro) diff --git a/lib/python3.1/site-packages/aiohttp/tracing.py b/lib/python3.1/site-packages/aiohttp/tracing.py new file mode 100644 index 0000000000000000000000000000000000000000..568fa7f9e38090e0c0a4738db4d51656ce31b99a --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/tracing.py @@ -0,0 +1,455 @@ +from types import SimpleNamespace +from typing import TYPE_CHECKING, Mapping, Optional, Type, TypeVar + +import attr +from aiosignal import Signal +from multidict import CIMultiDict +from yarl import URL + +from .client_reqrep import ClientResponse + +if TYPE_CHECKING: + from .client import ClientSession + + _ParamT_contra = TypeVar("_ParamT_contra", contravariant=True) + _TracingSignal = Signal[ClientSession, SimpleNamespace, _ParamT_contra] + + +__all__ = ( + "TraceConfig", + "TraceRequestStartParams", + "TraceRequestEndParams", + "TraceRequestExceptionParams", + "TraceConnectionQueuedStartParams", + "TraceConnectionQueuedEndParams", + "TraceConnectionCreateStartParams", + "TraceConnectionCreateEndParams", + "TraceConnectionReuseconnParams", + "TraceDnsResolveHostStartParams", + "TraceDnsResolveHostEndParams", + "TraceDnsCacheHitParams", + "TraceDnsCacheMissParams", + "TraceRequestRedirectParams", + "TraceRequestChunkSentParams", + "TraceResponseChunkReceivedParams", + "TraceRequestHeadersSentParams", +) + + +class TraceConfig: + """First-class used to trace requests launched via ClientSession objects.""" + + def __init__( + self, trace_config_ctx_factory: Type[SimpleNamespace] = SimpleNamespace + ) -> None: + self._on_request_start: _TracingSignal[TraceRequestStartParams] = Signal(self) + self._on_request_chunk_sent: _TracingSignal[TraceRequestChunkSentParams] = ( + Signal(self) + ) + self._on_response_chunk_received: _TracingSignal[ + TraceResponseChunkReceivedParams + ] = Signal(self) + self._on_request_end: _TracingSignal[TraceRequestEndParams] = Signal(self) + self._on_request_exception: _TracingSignal[TraceRequestExceptionParams] = ( + Signal(self) + ) + self._on_request_redirect: _TracingSignal[TraceRequestRedirectParams] = Signal( + self + ) + self._on_connection_queued_start: _TracingSignal[ + TraceConnectionQueuedStartParams + ] = Signal(self) + self._on_connection_queued_end: _TracingSignal[ + TraceConnectionQueuedEndParams + ] = Signal(self) + self._on_connection_create_start: _TracingSignal[ + TraceConnectionCreateStartParams + ] = Signal(self) + self._on_connection_create_end: _TracingSignal[ + TraceConnectionCreateEndParams + ] = Signal(self) + self._on_connection_reuseconn: _TracingSignal[ + TraceConnectionReuseconnParams + ] = Signal(self) + self._on_dns_resolvehost_start: _TracingSignal[ + TraceDnsResolveHostStartParams + ] = Signal(self) + self._on_dns_resolvehost_end: _TracingSignal[TraceDnsResolveHostEndParams] = ( + Signal(self) + ) + self._on_dns_cache_hit: _TracingSignal[TraceDnsCacheHitParams] = Signal(self) + self._on_dns_cache_miss: _TracingSignal[TraceDnsCacheMissParams] = Signal(self) + self._on_request_headers_sent: _TracingSignal[TraceRequestHeadersSentParams] = ( + Signal(self) + ) + + self._trace_config_ctx_factory = trace_config_ctx_factory + + def trace_config_ctx( + self, trace_request_ctx: Optional[Mapping[str, str]] = None + ) -> SimpleNamespace: + """Return a new trace_config_ctx instance""" + return self._trace_config_ctx_factory(trace_request_ctx=trace_request_ctx) + + def freeze(self) -> None: + self._on_request_start.freeze() + self._on_request_chunk_sent.freeze() + self._on_response_chunk_received.freeze() + self._on_request_end.freeze() + self._on_request_exception.freeze() + self._on_request_redirect.freeze() + self._on_connection_queued_start.freeze() + self._on_connection_queued_end.freeze() + self._on_connection_create_start.freeze() + self._on_connection_create_end.freeze() + self._on_connection_reuseconn.freeze() + self._on_dns_resolvehost_start.freeze() + self._on_dns_resolvehost_end.freeze() + self._on_dns_cache_hit.freeze() + self._on_dns_cache_miss.freeze() + self._on_request_headers_sent.freeze() + + @property + def on_request_start(self) -> "_TracingSignal[TraceRequestStartParams]": + return self._on_request_start + + @property + def on_request_chunk_sent( + self, + ) -> "_TracingSignal[TraceRequestChunkSentParams]": + return self._on_request_chunk_sent + + @property + def on_response_chunk_received( + self, + ) -> "_TracingSignal[TraceResponseChunkReceivedParams]": + return self._on_response_chunk_received + + @property + def on_request_end(self) -> "_TracingSignal[TraceRequestEndParams]": + return self._on_request_end + + @property + def on_request_exception( + self, + ) -> "_TracingSignal[TraceRequestExceptionParams]": + return self._on_request_exception + + @property + def on_request_redirect( + self, + ) -> "_TracingSignal[TraceRequestRedirectParams]": + return self._on_request_redirect + + @property + def on_connection_queued_start( + self, + ) -> "_TracingSignal[TraceConnectionQueuedStartParams]": + return self._on_connection_queued_start + + @property + def on_connection_queued_end( + self, + ) -> "_TracingSignal[TraceConnectionQueuedEndParams]": + return self._on_connection_queued_end + + @property + def on_connection_create_start( + self, + ) -> "_TracingSignal[TraceConnectionCreateStartParams]": + return self._on_connection_create_start + + @property + def on_connection_create_end( + self, + ) -> "_TracingSignal[TraceConnectionCreateEndParams]": + return self._on_connection_create_end + + @property + def on_connection_reuseconn( + self, + ) -> "_TracingSignal[TraceConnectionReuseconnParams]": + return self._on_connection_reuseconn + + @property + def on_dns_resolvehost_start( + self, + ) -> "_TracingSignal[TraceDnsResolveHostStartParams]": + return self._on_dns_resolvehost_start + + @property + def on_dns_resolvehost_end( + self, + ) -> "_TracingSignal[TraceDnsResolveHostEndParams]": + return self._on_dns_resolvehost_end + + @property + def on_dns_cache_hit(self) -> "_TracingSignal[TraceDnsCacheHitParams]": + return self._on_dns_cache_hit + + @property + def on_dns_cache_miss(self) -> "_TracingSignal[TraceDnsCacheMissParams]": + return self._on_dns_cache_miss + + @property + def on_request_headers_sent( + self, + ) -> "_TracingSignal[TraceRequestHeadersSentParams]": + return self._on_request_headers_sent + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestStartParams: + """Parameters sent by the `on_request_start` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestChunkSentParams: + """Parameters sent by the `on_request_chunk_sent` signal""" + + method: str + url: URL + chunk: bytes + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceResponseChunkReceivedParams: + """Parameters sent by the `on_response_chunk_received` signal""" + + method: str + url: URL + chunk: bytes + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestEndParams: + """Parameters sent by the `on_request_end` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + response: ClientResponse + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestExceptionParams: + """Parameters sent by the `on_request_exception` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + exception: BaseException + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestRedirectParams: + """Parameters sent by the `on_request_redirect` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + response: ClientResponse + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionQueuedStartParams: + """Parameters sent by the `on_connection_queued_start` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionQueuedEndParams: + """Parameters sent by the `on_connection_queued_end` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionCreateStartParams: + """Parameters sent by the `on_connection_create_start` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionCreateEndParams: + """Parameters sent by the `on_connection_create_end` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceConnectionReuseconnParams: + """Parameters sent by the `on_connection_reuseconn` signal""" + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceDnsResolveHostStartParams: + """Parameters sent by the `on_dns_resolvehost_start` signal""" + + host: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceDnsResolveHostEndParams: + """Parameters sent by the `on_dns_resolvehost_end` signal""" + + host: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceDnsCacheHitParams: + """Parameters sent by the `on_dns_cache_hit` signal""" + + host: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceDnsCacheMissParams: + """Parameters sent by the `on_dns_cache_miss` signal""" + + host: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class TraceRequestHeadersSentParams: + """Parameters sent by the `on_request_headers_sent` signal""" + + method: str + url: URL + headers: "CIMultiDict[str]" + + +class Trace: + """Internal dependency holder class. + + Used to keep together the main dependencies used + at the moment of send a signal. + """ + + def __init__( + self, + session: "ClientSession", + trace_config: TraceConfig, + trace_config_ctx: SimpleNamespace, + ) -> None: + self._trace_config = trace_config + self._trace_config_ctx = trace_config_ctx + self._session = session + + async def send_request_start( + self, method: str, url: URL, headers: "CIMultiDict[str]" + ) -> None: + return await self._trace_config.on_request_start.send( + self._session, + self._trace_config_ctx, + TraceRequestStartParams(method, url, headers), + ) + + async def send_request_chunk_sent( + self, method: str, url: URL, chunk: bytes + ) -> None: + return await self._trace_config.on_request_chunk_sent.send( + self._session, + self._trace_config_ctx, + TraceRequestChunkSentParams(method, url, chunk), + ) + + async def send_response_chunk_received( + self, method: str, url: URL, chunk: bytes + ) -> None: + return await self._trace_config.on_response_chunk_received.send( + self._session, + self._trace_config_ctx, + TraceResponseChunkReceivedParams(method, url, chunk), + ) + + async def send_request_end( + self, + method: str, + url: URL, + headers: "CIMultiDict[str]", + response: ClientResponse, + ) -> None: + return await self._trace_config.on_request_end.send( + self._session, + self._trace_config_ctx, + TraceRequestEndParams(method, url, headers, response), + ) + + async def send_request_exception( + self, + method: str, + url: URL, + headers: "CIMultiDict[str]", + exception: BaseException, + ) -> None: + return await self._trace_config.on_request_exception.send( + self._session, + self._trace_config_ctx, + TraceRequestExceptionParams(method, url, headers, exception), + ) + + async def send_request_redirect( + self, + method: str, + url: URL, + headers: "CIMultiDict[str]", + response: ClientResponse, + ) -> None: + return await self._trace_config._on_request_redirect.send( + self._session, + self._trace_config_ctx, + TraceRequestRedirectParams(method, url, headers, response), + ) + + async def send_connection_queued_start(self) -> None: + return await self._trace_config.on_connection_queued_start.send( + self._session, self._trace_config_ctx, TraceConnectionQueuedStartParams() + ) + + async def send_connection_queued_end(self) -> None: + return await self._trace_config.on_connection_queued_end.send( + self._session, self._trace_config_ctx, TraceConnectionQueuedEndParams() + ) + + async def send_connection_create_start(self) -> None: + return await self._trace_config.on_connection_create_start.send( + self._session, self._trace_config_ctx, TraceConnectionCreateStartParams() + ) + + async def send_connection_create_end(self) -> None: + return await self._trace_config.on_connection_create_end.send( + self._session, self._trace_config_ctx, TraceConnectionCreateEndParams() + ) + + async def send_connection_reuseconn(self) -> None: + return await self._trace_config.on_connection_reuseconn.send( + self._session, self._trace_config_ctx, TraceConnectionReuseconnParams() + ) + + async def send_dns_resolvehost_start(self, host: str) -> None: + return await self._trace_config.on_dns_resolvehost_start.send( + self._session, self._trace_config_ctx, TraceDnsResolveHostStartParams(host) + ) + + async def send_dns_resolvehost_end(self, host: str) -> None: + return await self._trace_config.on_dns_resolvehost_end.send( + self._session, self._trace_config_ctx, TraceDnsResolveHostEndParams(host) + ) + + async def send_dns_cache_hit(self, host: str) -> None: + return await self._trace_config.on_dns_cache_hit.send( + self._session, self._trace_config_ctx, TraceDnsCacheHitParams(host) + ) + + async def send_dns_cache_miss(self, host: str) -> None: + return await self._trace_config.on_dns_cache_miss.send( + self._session, self._trace_config_ctx, TraceDnsCacheMissParams(host) + ) + + async def send_request_headers( + self, method: str, url: URL, headers: "CIMultiDict[str]" + ) -> None: + return await self._trace_config._on_request_headers_sent.send( + self._session, + self._trace_config_ctx, + TraceRequestHeadersSentParams(method, url, headers), + ) diff --git a/lib/python3.1/site-packages/aiohttp/typedefs.py b/lib/python3.1/site-packages/aiohttp/typedefs.py new file mode 100644 index 0000000000000000000000000000000000000000..cc8c0825b4e522f7d1b6cf0058564f322fb5a905 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/typedefs.py @@ -0,0 +1,69 @@ +import json +import os +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Iterable, + Mapping, + Protocol, + Tuple, + Union, +) + +from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy, istr +from yarl import URL, Query as _Query + +Query = _Query + +DEFAULT_JSON_ENCODER = json.dumps +DEFAULT_JSON_DECODER = json.loads + +if TYPE_CHECKING: + _CIMultiDict = CIMultiDict[str] + _CIMultiDictProxy = CIMultiDictProxy[str] + _MultiDict = MultiDict[str] + _MultiDictProxy = MultiDictProxy[str] + from http.cookies import BaseCookie, Morsel + + from .web import Request, StreamResponse +else: + _CIMultiDict = CIMultiDict + _CIMultiDictProxy = CIMultiDictProxy + _MultiDict = MultiDict + _MultiDictProxy = MultiDictProxy + +Byteish = Union[bytes, bytearray, memoryview] +JSONEncoder = Callable[[Any], str] +JSONDecoder = Callable[[str], Any] +LooseHeaders = Union[ + Mapping[str, str], + Mapping[istr, str], + _CIMultiDict, + _CIMultiDictProxy, + Iterable[Tuple[Union[str, istr], str]], +] +RawHeaders = Tuple[Tuple[bytes, bytes], ...] +StrOrURL = Union[str, URL] + +LooseCookiesMappings = Mapping[str, Union[str, "BaseCookie[str]", "Morsel[Any]"]] +LooseCookiesIterables = Iterable[ + Tuple[str, Union[str, "BaseCookie[str]", "Morsel[Any]"]] +] +LooseCookies = Union[ + LooseCookiesMappings, + LooseCookiesIterables, + "BaseCookie[str]", +] + +Handler = Callable[["Request"], Awaitable["StreamResponse"]] + + +class Middleware(Protocol): + def __call__( + self, request: "Request", handler: Handler + ) -> Awaitable["StreamResponse"]: ... + + +PathLike = Union[str, "os.PathLike[str]"] diff --git a/lib/python3.1/site-packages/aiohttp/web.py b/lib/python3.1/site-packages/aiohttp/web.py new file mode 100644 index 0000000000000000000000000000000000000000..8307ff405caa9cfcad90da96713a0477cd9a1608 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web.py @@ -0,0 +1,605 @@ +import asyncio +import logging +import os +import socket +import sys +import warnings +from argparse import ArgumentParser +from collections.abc import Iterable +from contextlib import suppress +from importlib import import_module +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Iterable as TypingIterable, + List, + Optional, + Set, + Type, + Union, + cast, +) + +from .abc import AbstractAccessLogger +from .helpers import AppKey as AppKey +from .log import access_logger +from .typedefs import PathLike +from .web_app import Application as Application, CleanupError as CleanupError +from .web_exceptions import ( + HTTPAccepted as HTTPAccepted, + HTTPBadGateway as HTTPBadGateway, + HTTPBadRequest as HTTPBadRequest, + HTTPClientError as HTTPClientError, + HTTPConflict as HTTPConflict, + HTTPCreated as HTTPCreated, + HTTPError as HTTPError, + HTTPException as HTTPException, + HTTPExpectationFailed as HTTPExpectationFailed, + HTTPFailedDependency as HTTPFailedDependency, + HTTPForbidden as HTTPForbidden, + HTTPFound as HTTPFound, + HTTPGatewayTimeout as HTTPGatewayTimeout, + HTTPGone as HTTPGone, + HTTPInsufficientStorage as HTTPInsufficientStorage, + HTTPInternalServerError as HTTPInternalServerError, + HTTPLengthRequired as HTTPLengthRequired, + HTTPMethodNotAllowed as HTTPMethodNotAllowed, + HTTPMisdirectedRequest as HTTPMisdirectedRequest, + HTTPMove as HTTPMove, + HTTPMovedPermanently as HTTPMovedPermanently, + HTTPMultipleChoices as HTTPMultipleChoices, + HTTPNetworkAuthenticationRequired as HTTPNetworkAuthenticationRequired, + HTTPNoContent as HTTPNoContent, + HTTPNonAuthoritativeInformation as HTTPNonAuthoritativeInformation, + HTTPNotAcceptable as HTTPNotAcceptable, + HTTPNotExtended as HTTPNotExtended, + HTTPNotFound as HTTPNotFound, + HTTPNotImplemented as HTTPNotImplemented, + HTTPNotModified as HTTPNotModified, + HTTPOk as HTTPOk, + HTTPPartialContent as HTTPPartialContent, + HTTPPaymentRequired as HTTPPaymentRequired, + HTTPPermanentRedirect as HTTPPermanentRedirect, + HTTPPreconditionFailed as HTTPPreconditionFailed, + HTTPPreconditionRequired as HTTPPreconditionRequired, + HTTPProxyAuthenticationRequired as HTTPProxyAuthenticationRequired, + HTTPRedirection as HTTPRedirection, + HTTPRequestEntityTooLarge as HTTPRequestEntityTooLarge, + HTTPRequestHeaderFieldsTooLarge as HTTPRequestHeaderFieldsTooLarge, + HTTPRequestRangeNotSatisfiable as HTTPRequestRangeNotSatisfiable, + HTTPRequestTimeout as HTTPRequestTimeout, + HTTPRequestURITooLong as HTTPRequestURITooLong, + HTTPResetContent as HTTPResetContent, + HTTPSeeOther as HTTPSeeOther, + HTTPServerError as HTTPServerError, + HTTPServiceUnavailable as HTTPServiceUnavailable, + HTTPSuccessful as HTTPSuccessful, + HTTPTemporaryRedirect as HTTPTemporaryRedirect, + HTTPTooManyRequests as HTTPTooManyRequests, + HTTPUnauthorized as HTTPUnauthorized, + HTTPUnavailableForLegalReasons as HTTPUnavailableForLegalReasons, + HTTPUnprocessableEntity as HTTPUnprocessableEntity, + HTTPUnsupportedMediaType as HTTPUnsupportedMediaType, + HTTPUpgradeRequired as HTTPUpgradeRequired, + HTTPUseProxy as HTTPUseProxy, + HTTPVariantAlsoNegotiates as HTTPVariantAlsoNegotiates, + HTTPVersionNotSupported as HTTPVersionNotSupported, + NotAppKeyWarning as NotAppKeyWarning, +) +from .web_fileresponse import FileResponse as FileResponse +from .web_log import AccessLogger +from .web_middlewares import ( + middleware as middleware, + normalize_path_middleware as normalize_path_middleware, +) +from .web_protocol import ( + PayloadAccessError as PayloadAccessError, + RequestHandler as RequestHandler, + RequestPayloadError as RequestPayloadError, +) +from .web_request import ( + BaseRequest as BaseRequest, + FileField as FileField, + Request as Request, +) +from .web_response import ( + ContentCoding as ContentCoding, + Response as Response, + StreamResponse as StreamResponse, + json_response as json_response, +) +from .web_routedef import ( + AbstractRouteDef as AbstractRouteDef, + RouteDef as RouteDef, + RouteTableDef as RouteTableDef, + StaticDef as StaticDef, + delete as delete, + get as get, + head as head, + options as options, + patch as patch, + post as post, + put as put, + route as route, + static as static, + view as view, +) +from .web_runner import ( + AppRunner as AppRunner, + BaseRunner as BaseRunner, + BaseSite as BaseSite, + GracefulExit as GracefulExit, + NamedPipeSite as NamedPipeSite, + ServerRunner as ServerRunner, + SockSite as SockSite, + TCPSite as TCPSite, + UnixSite as UnixSite, +) +from .web_server import Server as Server +from .web_urldispatcher import ( + AbstractResource as AbstractResource, + AbstractRoute as AbstractRoute, + DynamicResource as DynamicResource, + PlainResource as PlainResource, + PrefixedSubAppResource as PrefixedSubAppResource, + Resource as Resource, + ResourceRoute as ResourceRoute, + StaticResource as StaticResource, + UrlDispatcher as UrlDispatcher, + UrlMappingMatchInfo as UrlMappingMatchInfo, + View as View, +) +from .web_ws import ( + WebSocketReady as WebSocketReady, + WebSocketResponse as WebSocketResponse, + WSMsgType as WSMsgType, +) + +__all__ = ( + # web_app + "AppKey", + "Application", + "CleanupError", + # web_exceptions + "NotAppKeyWarning", + "HTTPAccepted", + "HTTPBadGateway", + "HTTPBadRequest", + "HTTPClientError", + "HTTPConflict", + "HTTPCreated", + "HTTPError", + "HTTPException", + "HTTPExpectationFailed", + "HTTPFailedDependency", + "HTTPForbidden", + "HTTPFound", + "HTTPGatewayTimeout", + "HTTPGone", + "HTTPInsufficientStorage", + "HTTPInternalServerError", + "HTTPLengthRequired", + "HTTPMethodNotAllowed", + "HTTPMisdirectedRequest", + "HTTPMove", + "HTTPMovedPermanently", + "HTTPMultipleChoices", + "HTTPNetworkAuthenticationRequired", + "HTTPNoContent", + "HTTPNonAuthoritativeInformation", + "HTTPNotAcceptable", + "HTTPNotExtended", + "HTTPNotFound", + "HTTPNotImplemented", + "HTTPNotModified", + "HTTPOk", + "HTTPPartialContent", + "HTTPPaymentRequired", + "HTTPPermanentRedirect", + "HTTPPreconditionFailed", + "HTTPPreconditionRequired", + "HTTPProxyAuthenticationRequired", + "HTTPRedirection", + "HTTPRequestEntityTooLarge", + "HTTPRequestHeaderFieldsTooLarge", + "HTTPRequestRangeNotSatisfiable", + "HTTPRequestTimeout", + "HTTPRequestURITooLong", + "HTTPResetContent", + "HTTPSeeOther", + "HTTPServerError", + "HTTPServiceUnavailable", + "HTTPSuccessful", + "HTTPTemporaryRedirect", + "HTTPTooManyRequests", + "HTTPUnauthorized", + "HTTPUnavailableForLegalReasons", + "HTTPUnprocessableEntity", + "HTTPUnsupportedMediaType", + "HTTPUpgradeRequired", + "HTTPUseProxy", + "HTTPVariantAlsoNegotiates", + "HTTPVersionNotSupported", + # web_fileresponse + "FileResponse", + # web_middlewares + "middleware", + "normalize_path_middleware", + # web_protocol + "PayloadAccessError", + "RequestHandler", + "RequestPayloadError", + # web_request + "BaseRequest", + "FileField", + "Request", + # web_response + "ContentCoding", + "Response", + "StreamResponse", + "json_response", + # web_routedef + "AbstractRouteDef", + "RouteDef", + "RouteTableDef", + "StaticDef", + "delete", + "get", + "head", + "options", + "patch", + "post", + "put", + "route", + "static", + "view", + # web_runner + "AppRunner", + "BaseRunner", + "BaseSite", + "GracefulExit", + "ServerRunner", + "SockSite", + "TCPSite", + "UnixSite", + "NamedPipeSite", + # web_server + "Server", + # web_urldispatcher + "AbstractResource", + "AbstractRoute", + "DynamicResource", + "PlainResource", + "PrefixedSubAppResource", + "Resource", + "ResourceRoute", + "StaticResource", + "UrlDispatcher", + "UrlMappingMatchInfo", + "View", + # web_ws + "WebSocketReady", + "WebSocketResponse", + "WSMsgType", + # web + "run_app", +) + + +if TYPE_CHECKING: + from ssl import SSLContext +else: + try: + from ssl import SSLContext + except ImportError: # pragma: no cover + SSLContext = object # type: ignore[misc,assignment] + +# Only display warning when using -Wdefault, -We, -X dev or similar. +warnings.filterwarnings("ignore", category=NotAppKeyWarning, append=True) + +HostSequence = TypingIterable[str] + + +async def _run_app( + app: Union[Application, Awaitable[Application]], + *, + host: Optional[Union[str, HostSequence]] = None, + port: Optional[int] = None, + path: Union[PathLike, TypingIterable[PathLike], None] = None, + sock: Optional[Union[socket.socket, TypingIterable[socket.socket]]] = None, + shutdown_timeout: float = 60.0, + keepalive_timeout: float = 75.0, + ssl_context: Optional[SSLContext] = None, + print: Optional[Callable[..., None]] = print, + backlog: int = 128, + access_log_class: Type[AbstractAccessLogger] = AccessLogger, + access_log_format: str = AccessLogger.LOG_FORMAT, + access_log: Optional[logging.Logger] = access_logger, + handle_signals: bool = True, + reuse_address: Optional[bool] = None, + reuse_port: Optional[bool] = None, + handler_cancellation: bool = False, +) -> None: + # An internal function to actually do all dirty job for application running + if asyncio.iscoroutine(app): + app = await app + + app = cast(Application, app) + + runner = AppRunner( + app, + handle_signals=handle_signals, + access_log_class=access_log_class, + access_log_format=access_log_format, + access_log=access_log, + keepalive_timeout=keepalive_timeout, + shutdown_timeout=shutdown_timeout, + handler_cancellation=handler_cancellation, + ) + + await runner.setup() + + sites: List[BaseSite] = [] + + try: + if host is not None: + if isinstance(host, str): + sites.append( + TCPSite( + runner, + host, + port, + ssl_context=ssl_context, + backlog=backlog, + reuse_address=reuse_address, + reuse_port=reuse_port, + ) + ) + else: + for h in host: + sites.append( + TCPSite( + runner, + h, + port, + ssl_context=ssl_context, + backlog=backlog, + reuse_address=reuse_address, + reuse_port=reuse_port, + ) + ) + elif path is None and sock is None or port is not None: + sites.append( + TCPSite( + runner, + port=port, + ssl_context=ssl_context, + backlog=backlog, + reuse_address=reuse_address, + reuse_port=reuse_port, + ) + ) + + if path is not None: + if isinstance(path, (str, os.PathLike)): + sites.append( + UnixSite( + runner, + path, + ssl_context=ssl_context, + backlog=backlog, + ) + ) + else: + for p in path: + sites.append( + UnixSite( + runner, + p, + ssl_context=ssl_context, + backlog=backlog, + ) + ) + + if sock is not None: + if not isinstance(sock, Iterable): + sites.append( + SockSite( + runner, + sock, + ssl_context=ssl_context, + backlog=backlog, + ) + ) + else: + for s in sock: + sites.append( + SockSite( + runner, + s, + ssl_context=ssl_context, + backlog=backlog, + ) + ) + for site in sites: + await site.start() + + if print: # pragma: no branch + names = sorted(str(s.name) for s in runner.sites) + print( + "======== Running on {} ========\n" + "(Press CTRL+C to quit)".format(", ".join(names)) + ) + + # sleep forever by 1 hour intervals, + while True: + await asyncio.sleep(3600) + finally: + await runner.cleanup() + + +def _cancel_tasks( + to_cancel: Set["asyncio.Task[Any]"], loop: asyncio.AbstractEventLoop +) -> None: + if not to_cancel: + return + + for task in to_cancel: + task.cancel() + + loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True)) + + for task in to_cancel: + if task.cancelled(): + continue + if task.exception() is not None: + loop.call_exception_handler( + { + "message": "unhandled exception during asyncio.run() shutdown", + "exception": task.exception(), + "task": task, + } + ) + + +def run_app( + app: Union[Application, Awaitable[Application]], + *, + host: Optional[Union[str, HostSequence]] = None, + port: Optional[int] = None, + path: Union[PathLike, TypingIterable[PathLike], None] = None, + sock: Optional[Union[socket.socket, TypingIterable[socket.socket]]] = None, + shutdown_timeout: float = 60.0, + keepalive_timeout: float = 75.0, + ssl_context: Optional[SSLContext] = None, + print: Optional[Callable[..., None]] = print, + backlog: int = 128, + access_log_class: Type[AbstractAccessLogger] = AccessLogger, + access_log_format: str = AccessLogger.LOG_FORMAT, + access_log: Optional[logging.Logger] = access_logger, + handle_signals: bool = True, + reuse_address: Optional[bool] = None, + reuse_port: Optional[bool] = None, + handler_cancellation: bool = False, + loop: Optional[asyncio.AbstractEventLoop] = None, +) -> None: + """Run an app locally""" + if loop is None: + loop = asyncio.new_event_loop() + + # Configure if and only if in debugging mode and using the default logger + if loop.get_debug() and access_log and access_log.name == "aiohttp.access": + if access_log.level == logging.NOTSET: + access_log.setLevel(logging.DEBUG) + if not access_log.hasHandlers(): + access_log.addHandler(logging.StreamHandler()) + + main_task = loop.create_task( + _run_app( + app, + host=host, + port=port, + path=path, + sock=sock, + shutdown_timeout=shutdown_timeout, + keepalive_timeout=keepalive_timeout, + ssl_context=ssl_context, + print=print, + backlog=backlog, + access_log_class=access_log_class, + access_log_format=access_log_format, + access_log=access_log, + handle_signals=handle_signals, + reuse_address=reuse_address, + reuse_port=reuse_port, + handler_cancellation=handler_cancellation, + ) + ) + + try: + asyncio.set_event_loop(loop) + loop.run_until_complete(main_task) + except (GracefulExit, KeyboardInterrupt): # pragma: no cover + pass + finally: + try: + main_task.cancel() + with suppress(asyncio.CancelledError): + loop.run_until_complete(main_task) + finally: + _cancel_tasks(asyncio.all_tasks(loop), loop) + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + + +def main(argv: List[str]) -> None: + arg_parser = ArgumentParser( + description="aiohttp.web Application server", prog="aiohttp.web" + ) + arg_parser.add_argument( + "entry_func", + help=( + "Callable returning the `aiohttp.web.Application` instance to " + "run. Should be specified in the 'module:function' syntax." + ), + metavar="entry-func", + ) + arg_parser.add_argument( + "-H", + "--hostname", + help="TCP/IP hostname to serve on (default: localhost)", + default=None, + ) + arg_parser.add_argument( + "-P", + "--port", + help="TCP/IP port to serve on (default: %(default)r)", + type=int, + default=8080, + ) + arg_parser.add_argument( + "-U", + "--path", + help="Unix file system path to serve on. Can be combined with hostname " + "to serve on both Unix and TCP.", + ) + args, extra_argv = arg_parser.parse_known_args(argv) + + # Import logic + mod_str, _, func_str = args.entry_func.partition(":") + if not func_str or not mod_str: + arg_parser.error("'entry-func' not in 'module:function' syntax") + if mod_str.startswith("."): + arg_parser.error("relative module names not supported") + try: + module = import_module(mod_str) + except ImportError as ex: + arg_parser.error(f"unable to import {mod_str}: {ex}") + try: + func = getattr(module, func_str) + except AttributeError: + arg_parser.error(f"module {mod_str!r} has no attribute {func_str!r}") + + # Compatibility logic + if args.path is not None and not hasattr(socket, "AF_UNIX"): + arg_parser.error( + "file system paths not supported by your operating environment" + ) + + logging.basicConfig(level=logging.DEBUG) + + if args.path and args.hostname is None: + host = port = None + else: + host = args.hostname or "localhost" + port = args.port + + app = func(extra_argv) + run_app(app, host=host, port=port, path=args.path) + arg_parser.exit(message="Stopped\n") + + +if __name__ == "__main__": # pragma: no branch + main(sys.argv[1:]) # pragma: no cover diff --git a/lib/python3.1/site-packages/aiohttp/web_app.py b/lib/python3.1/site-packages/aiohttp/web_app.py new file mode 100644 index 0000000000000000000000000000000000000000..619c0085da1985b97f7e222d30c1870cb010a128 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web_app.py @@ -0,0 +1,620 @@ +import asyncio +import logging +import warnings +from functools import lru_cache, partial, update_wrapper +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterator, + Awaitable, + Callable, + Dict, + Iterable, + Iterator, + List, + Mapping, + MutableMapping, + Optional, + Sequence, + Tuple, + Type, + TypeVar, + Union, + cast, + overload, +) + +from aiosignal import Signal +from frozenlist import FrozenList + +from . import hdrs +from .abc import ( + AbstractAccessLogger, + AbstractMatchInfo, + AbstractRouter, + AbstractStreamWriter, +) +from .helpers import DEBUG, AppKey +from .http_parser import RawRequestMessage +from .log import web_logger +from .streams import StreamReader +from .typedefs import Handler, Middleware +from .web_exceptions import NotAppKeyWarning +from .web_log import AccessLogger +from .web_middlewares import _fix_request_current_app +from .web_protocol import RequestHandler +from .web_request import Request +from .web_response import StreamResponse +from .web_routedef import AbstractRouteDef +from .web_server import Server +from .web_urldispatcher import ( + AbstractResource, + AbstractRoute, + Domain, + MaskDomain, + MatchedSubAppResource, + PrefixedSubAppResource, + SystemRoute, + UrlDispatcher, +) + +__all__ = ("Application", "CleanupError") + + +if TYPE_CHECKING: + _AppSignal = Signal["Application"] + _RespPrepareSignal = Signal[Request, StreamResponse] + _Middlewares = FrozenList[Middleware] + _MiddlewaresHandlers = Optional[Sequence[Tuple[Middleware, bool]]] + _Subapps = List["Application"] +else: + # No type checker mode, skip types + _AppSignal = Signal + _RespPrepareSignal = Signal + _Middlewares = FrozenList + _MiddlewaresHandlers = Optional[Sequence] + _Subapps = List + +_T = TypeVar("_T") +_U = TypeVar("_U") +_Resource = TypeVar("_Resource", bound=AbstractResource) + + +def _build_middlewares( + handler: Handler, apps: Tuple["Application", ...] +) -> Callable[[Request], Awaitable[StreamResponse]]: + """Apply middlewares to handler.""" + for app in apps[::-1]: + for m, _ in app._middlewares_handlers: # type: ignore[union-attr] + handler = update_wrapper(partial(m, handler=handler), handler) + return handler + + +_cached_build_middleware = lru_cache(maxsize=1024)(_build_middlewares) + + +class Application(MutableMapping[Union[str, AppKey[Any]], Any]): + ATTRS = frozenset( + [ + "logger", + "_debug", + "_router", + "_loop", + "_handler_args", + "_middlewares", + "_middlewares_handlers", + "_has_legacy_middlewares", + "_run_middlewares", + "_state", + "_frozen", + "_pre_frozen", + "_subapps", + "_on_response_prepare", + "_on_startup", + "_on_shutdown", + "_on_cleanup", + "_client_max_size", + "_cleanup_ctx", + ] + ) + + def __init__( + self, + *, + logger: logging.Logger = web_logger, + router: Optional[UrlDispatcher] = None, + middlewares: Iterable[Middleware] = (), + handler_args: Optional[Mapping[str, Any]] = None, + client_max_size: int = 1024**2, + loop: Optional[asyncio.AbstractEventLoop] = None, + debug: Any = ..., # mypy doesn't support ellipsis + ) -> None: + if router is None: + router = UrlDispatcher() + else: + warnings.warn( + "router argument is deprecated", DeprecationWarning, stacklevel=2 + ) + assert isinstance(router, AbstractRouter), router + + if loop is not None: + warnings.warn( + "loop argument is deprecated", DeprecationWarning, stacklevel=2 + ) + + if debug is not ...: + warnings.warn( + "debug argument is deprecated", DeprecationWarning, stacklevel=2 + ) + self._debug = debug + self._router: UrlDispatcher = router + self._loop = loop + self._handler_args = handler_args + self.logger = logger + + self._middlewares: _Middlewares = FrozenList(middlewares) + + # initialized on freezing + self._middlewares_handlers: _MiddlewaresHandlers = None + # initialized on freezing + self._run_middlewares: Optional[bool] = None + self._has_legacy_middlewares: bool = True + + self._state: Dict[Union[AppKey[Any], str], object] = {} + self._frozen = False + self._pre_frozen = False + self._subapps: _Subapps = [] + + self._on_response_prepare: _RespPrepareSignal = Signal(self) + self._on_startup: _AppSignal = Signal(self) + self._on_shutdown: _AppSignal = Signal(self) + self._on_cleanup: _AppSignal = Signal(self) + self._cleanup_ctx = CleanupContext() + self._on_startup.append(self._cleanup_ctx._on_startup) + self._on_cleanup.append(self._cleanup_ctx._on_cleanup) + self._client_max_size = client_max_size + + def __init_subclass__(cls: Type["Application"]) -> None: + warnings.warn( + "Inheritance class {} from web.Application " + "is discouraged".format(cls.__name__), + DeprecationWarning, + stacklevel=3, + ) + + if DEBUG: # pragma: no cover + + def __setattr__(self, name: str, val: Any) -> None: + if name not in self.ATTRS: + warnings.warn( + "Setting custom web.Application.{} attribute " + "is discouraged".format(name), + DeprecationWarning, + stacklevel=2, + ) + super().__setattr__(name, val) + + # MutableMapping API + + def __eq__(self, other: object) -> bool: + return self is other + + @overload # type: ignore[override] + def __getitem__(self, key: AppKey[_T]) -> _T: ... + + @overload + def __getitem__(self, key: str) -> Any: ... + + def __getitem__(self, key: Union[str, AppKey[_T]]) -> Any: + return self._state[key] + + def _check_frozen(self) -> None: + if self._frozen: + warnings.warn( + "Changing state of started or joined application is deprecated", + DeprecationWarning, + stacklevel=3, + ) + + @overload # type: ignore[override] + def __setitem__(self, key: AppKey[_T], value: _T) -> None: ... + + @overload + def __setitem__(self, key: str, value: Any) -> None: ... + + def __setitem__(self, key: Union[str, AppKey[_T]], value: Any) -> None: + self._check_frozen() + if not isinstance(key, AppKey): + warnings.warn( + "It is recommended to use web.AppKey instances for keys.\n" + + "https://docs.aiohttp.org/en/stable/web_advanced.html" + + "#application-s-config", + category=NotAppKeyWarning, + stacklevel=2, + ) + self._state[key] = value + + def __delitem__(self, key: Union[str, AppKey[_T]]) -> None: + self._check_frozen() + del self._state[key] + + def __len__(self) -> int: + return len(self._state) + + def __iter__(self) -> Iterator[Union[str, AppKey[Any]]]: + return iter(self._state) + + def __hash__(self) -> int: + return id(self) + + @overload # type: ignore[override] + def get(self, key: AppKey[_T], default: None = ...) -> Optional[_T]: ... + + @overload + def get(self, key: AppKey[_T], default: _U) -> Union[_T, _U]: ... + + @overload + def get(self, key: str, default: Any = ...) -> Any: ... + + def get(self, key: Union[str, AppKey[_T]], default: Any = None) -> Any: + return self._state.get(key, default) + + ######## + @property + def loop(self) -> asyncio.AbstractEventLoop: + # Technically the loop can be None + # but we mask it by explicit type cast + # to provide more convenient type annotation + warnings.warn("loop property is deprecated", DeprecationWarning, stacklevel=2) + return cast(asyncio.AbstractEventLoop, self._loop) + + def _set_loop(self, loop: Optional[asyncio.AbstractEventLoop]) -> None: + if loop is None: + loop = asyncio.get_event_loop() + if self._loop is not None and self._loop is not loop: + raise RuntimeError( + "web.Application instance initialized with different loop" + ) + + self._loop = loop + + # set loop debug + if self._debug is ...: + self._debug = loop.get_debug() + + # set loop to sub applications + for subapp in self._subapps: + subapp._set_loop(loop) + + @property + def pre_frozen(self) -> bool: + return self._pre_frozen + + def pre_freeze(self) -> None: + if self._pre_frozen: + return + + self._pre_frozen = True + self._middlewares.freeze() + self._router.freeze() + self._on_response_prepare.freeze() + self._cleanup_ctx.freeze() + self._on_startup.freeze() + self._on_shutdown.freeze() + self._on_cleanup.freeze() + self._middlewares_handlers = tuple(self._prepare_middleware()) + self._has_legacy_middlewares = any( + not new_style for _, new_style in self._middlewares_handlers + ) + + # If current app and any subapp do not have middlewares avoid run all + # of the code footprint that it implies, which have a middleware + # hardcoded per app that sets up the current_app attribute. If no + # middlewares are configured the handler will receive the proper + # current_app without needing all of this code. + self._run_middlewares = True if self.middlewares else False + + for subapp in self._subapps: + subapp.pre_freeze() + self._run_middlewares = self._run_middlewares or subapp._run_middlewares + + @property + def frozen(self) -> bool: + return self._frozen + + def freeze(self) -> None: + if self._frozen: + return + + self.pre_freeze() + self._frozen = True + for subapp in self._subapps: + subapp.freeze() + + @property + def debug(self) -> bool: + warnings.warn("debug property is deprecated", DeprecationWarning, stacklevel=2) + return self._debug # type: ignore[no-any-return] + + def _reg_subapp_signals(self, subapp: "Application") -> None: + def reg_handler(signame: str) -> None: + subsig = getattr(subapp, signame) + + async def handler(app: "Application") -> None: + await subsig.send(subapp) + + appsig = getattr(self, signame) + appsig.append(handler) + + reg_handler("on_startup") + reg_handler("on_shutdown") + reg_handler("on_cleanup") + + def add_subapp(self, prefix: str, subapp: "Application") -> PrefixedSubAppResource: + if not isinstance(prefix, str): + raise TypeError("Prefix must be str") + prefix = prefix.rstrip("/") + if not prefix: + raise ValueError("Prefix cannot be empty") + factory = partial(PrefixedSubAppResource, prefix, subapp) + return self._add_subapp(factory, subapp) + + def _add_subapp( + self, resource_factory: Callable[[], _Resource], subapp: "Application" + ) -> _Resource: + if self.frozen: + raise RuntimeError("Cannot add sub application to frozen application") + if subapp.frozen: + raise RuntimeError("Cannot add frozen application") + resource = resource_factory() + self.router.register_resource(resource) + self._reg_subapp_signals(subapp) + self._subapps.append(subapp) + subapp.pre_freeze() + if self._loop is not None: + subapp._set_loop(self._loop) + return resource + + def add_domain(self, domain: str, subapp: "Application") -> MatchedSubAppResource: + if not isinstance(domain, str): + raise TypeError("Domain must be str") + elif "*" in domain: + rule: Domain = MaskDomain(domain) + else: + rule = Domain(domain) + factory = partial(MatchedSubAppResource, rule, subapp) + return self._add_subapp(factory, subapp) + + def add_routes(self, routes: Iterable[AbstractRouteDef]) -> List[AbstractRoute]: + return self.router.add_routes(routes) + + @property + def on_response_prepare(self) -> _RespPrepareSignal: + return self._on_response_prepare + + @property + def on_startup(self) -> _AppSignal: + return self._on_startup + + @property + def on_shutdown(self) -> _AppSignal: + return self._on_shutdown + + @property + def on_cleanup(self) -> _AppSignal: + return self._on_cleanup + + @property + def cleanup_ctx(self) -> "CleanupContext": + return self._cleanup_ctx + + @property + def router(self) -> UrlDispatcher: + return self._router + + @property + def middlewares(self) -> _Middlewares: + return self._middlewares + + def _make_handler( + self, + *, + loop: Optional[asyncio.AbstractEventLoop] = None, + access_log_class: Type[AbstractAccessLogger] = AccessLogger, + **kwargs: Any, + ) -> Server: + + if not issubclass(access_log_class, AbstractAccessLogger): + raise TypeError( + "access_log_class must be subclass of " + "aiohttp.abc.AbstractAccessLogger, got {}".format(access_log_class) + ) + + self._set_loop(loop) + self.freeze() + + kwargs["debug"] = self._debug + kwargs["access_log_class"] = access_log_class + if self._handler_args: + for k, v in self._handler_args.items(): + kwargs[k] = v + + return Server( + self._handle, # type: ignore[arg-type] + request_factory=self._make_request, + loop=self._loop, + **kwargs, + ) + + def make_handler( + self, + *, + loop: Optional[asyncio.AbstractEventLoop] = None, + access_log_class: Type[AbstractAccessLogger] = AccessLogger, + **kwargs: Any, + ) -> Server: + + warnings.warn( + "Application.make_handler(...) is deprecated, use AppRunner API instead", + DeprecationWarning, + stacklevel=2, + ) + + return self._make_handler( + loop=loop, access_log_class=access_log_class, **kwargs + ) + + async def startup(self) -> None: + """Causes on_startup signal + + Should be called in the event loop along with the request handler. + """ + await self.on_startup.send(self) + + async def shutdown(self) -> None: + """Causes on_shutdown signal + + Should be called before cleanup() + """ + await self.on_shutdown.send(self) + + async def cleanup(self) -> None: + """Causes on_cleanup signal + + Should be called after shutdown() + """ + if self.on_cleanup.frozen: + await self.on_cleanup.send(self) + else: + # If an exception occurs in startup, ensure cleanup contexts are completed. + await self._cleanup_ctx._on_cleanup(self) + + def _make_request( + self, + message: RawRequestMessage, + payload: StreamReader, + protocol: RequestHandler, + writer: AbstractStreamWriter, + task: "asyncio.Task[None]", + _cls: Type[Request] = Request, + ) -> Request: + if TYPE_CHECKING: + assert self._loop is not None + return _cls( + message, + payload, + protocol, + writer, + task, + self._loop, + client_max_size=self._client_max_size, + ) + + def _prepare_middleware(self) -> Iterator[Tuple[Middleware, bool]]: + for m in reversed(self._middlewares): + if getattr(m, "__middleware_version__", None) == 1: + yield m, True + else: + warnings.warn( + f'old-style middleware "{m!r}" deprecated, see #2252', + DeprecationWarning, + stacklevel=2, + ) + yield m, False + + yield _fix_request_current_app(self), True + + async def _handle(self, request: Request) -> StreamResponse: + loop = asyncio.get_event_loop() + debug = loop.get_debug() + match_info = await self._router.resolve(request) + if debug: # pragma: no cover + if not isinstance(match_info, AbstractMatchInfo): + raise TypeError( + "match_info should be AbstractMatchInfo " + "instance, not {!r}".format(match_info) + ) + match_info.add_app(self) + + match_info.freeze() + + request._match_info = match_info + + if request.headers.get(hdrs.EXPECT): + resp = await match_info.expect_handler(request) + await request.writer.drain() + if resp is not None: + return resp + + handler = match_info.handler + + if self._run_middlewares: + # If its a SystemRoute, don't cache building the middlewares since + # they are constructed for every MatchInfoError as a new handler + # is made each time. + if not self._has_legacy_middlewares and not isinstance( + match_info.route, SystemRoute + ): + handler = _cached_build_middleware(handler, match_info.apps) + else: + for app in match_info.apps[::-1]: + for m, new_style in app._middlewares_handlers: # type: ignore[union-attr] + if new_style: + handler = update_wrapper( + partial(m, handler=handler), handler + ) + else: + handler = await m(app, handler) # type: ignore[arg-type,assignment] + + return await handler(request) + + def __call__(self) -> "Application": + """gunicorn compatibility""" + return self + + def __repr__(self) -> str: + return f"" + + def __bool__(self) -> bool: + return True + + +class CleanupError(RuntimeError): + @property + def exceptions(self) -> List[BaseException]: + return cast(List[BaseException], self.args[1]) + + +if TYPE_CHECKING: + _CleanupContextBase = FrozenList[Callable[[Application], AsyncIterator[None]]] +else: + _CleanupContextBase = FrozenList + + +class CleanupContext(_CleanupContextBase): + def __init__(self) -> None: + super().__init__() + self._exits: List[AsyncIterator[None]] = [] + + async def _on_startup(self, app: Application) -> None: + for cb in self: + it = cb(app).__aiter__() + await it.__anext__() + self._exits.append(it) + + async def _on_cleanup(self, app: Application) -> None: + errors = [] + for it in reversed(self._exits): + try: + await it.__anext__() + except StopAsyncIteration: + pass + except (Exception, asyncio.CancelledError) as exc: + errors.append(exc) + else: + errors.append(RuntimeError(f"{it!r} has more than one 'yield'")) + if errors: + if len(errors) == 1: + raise errors[0] + else: + raise CleanupError("Multiple errors on cleanup stage", errors) diff --git a/lib/python3.1/site-packages/aiohttp/web_exceptions.py b/lib/python3.1/site-packages/aiohttp/web_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..ee2c1e72d40ac93c00cbfcef6c84888a336855a3 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web_exceptions.py @@ -0,0 +1,452 @@ +import warnings +from typing import Any, Dict, Iterable, List, Optional, Set # noqa + +from yarl import URL + +from .typedefs import LooseHeaders, StrOrURL +from .web_response import Response + +__all__ = ( + "HTTPException", + "HTTPError", + "HTTPRedirection", + "HTTPSuccessful", + "HTTPOk", + "HTTPCreated", + "HTTPAccepted", + "HTTPNonAuthoritativeInformation", + "HTTPNoContent", + "HTTPResetContent", + "HTTPPartialContent", + "HTTPMove", + "HTTPMultipleChoices", + "HTTPMovedPermanently", + "HTTPFound", + "HTTPSeeOther", + "HTTPNotModified", + "HTTPUseProxy", + "HTTPTemporaryRedirect", + "HTTPPermanentRedirect", + "HTTPClientError", + "HTTPBadRequest", + "HTTPUnauthorized", + "HTTPPaymentRequired", + "HTTPForbidden", + "HTTPNotFound", + "HTTPMethodNotAllowed", + "HTTPNotAcceptable", + "HTTPProxyAuthenticationRequired", + "HTTPRequestTimeout", + "HTTPConflict", + "HTTPGone", + "HTTPLengthRequired", + "HTTPPreconditionFailed", + "HTTPRequestEntityTooLarge", + "HTTPRequestURITooLong", + "HTTPUnsupportedMediaType", + "HTTPRequestRangeNotSatisfiable", + "HTTPExpectationFailed", + "HTTPMisdirectedRequest", + "HTTPUnprocessableEntity", + "HTTPFailedDependency", + "HTTPUpgradeRequired", + "HTTPPreconditionRequired", + "HTTPTooManyRequests", + "HTTPRequestHeaderFieldsTooLarge", + "HTTPUnavailableForLegalReasons", + "HTTPServerError", + "HTTPInternalServerError", + "HTTPNotImplemented", + "HTTPBadGateway", + "HTTPServiceUnavailable", + "HTTPGatewayTimeout", + "HTTPVersionNotSupported", + "HTTPVariantAlsoNegotiates", + "HTTPInsufficientStorage", + "HTTPNotExtended", + "HTTPNetworkAuthenticationRequired", +) + + +class NotAppKeyWarning(UserWarning): + """Warning when not using AppKey in Application.""" + + +############################################################ +# HTTP Exceptions +############################################################ + + +class HTTPException(Response, Exception): + + # You should set in subclasses: + # status = 200 + + status_code = -1 + empty_body = False + + __http_exception__ = True + + def __init__( + self, + *, + headers: Optional[LooseHeaders] = None, + reason: Optional[str] = None, + body: Any = None, + text: Optional[str] = None, + content_type: Optional[str] = None, + ) -> None: + if body is not None: + warnings.warn( + "body argument is deprecated for http web exceptions", + DeprecationWarning, + ) + Response.__init__( + self, + status=self.status_code, + headers=headers, + reason=reason, + body=body, + text=text, + content_type=content_type, + ) + Exception.__init__(self, self.reason) + if self.body is None and not self.empty_body: + self.text = f"{self.status}: {self.reason}" + + def __bool__(self) -> bool: + return True + + +class HTTPError(HTTPException): + """Base class for exceptions with status codes in the 400s and 500s.""" + + +class HTTPRedirection(HTTPException): + """Base class for exceptions with status codes in the 300s.""" + + +class HTTPSuccessful(HTTPException): + """Base class for exceptions with status codes in the 200s.""" + + +class HTTPOk(HTTPSuccessful): + status_code = 200 + + +class HTTPCreated(HTTPSuccessful): + status_code = 201 + + +class HTTPAccepted(HTTPSuccessful): + status_code = 202 + + +class HTTPNonAuthoritativeInformation(HTTPSuccessful): + status_code = 203 + + +class HTTPNoContent(HTTPSuccessful): + status_code = 204 + empty_body = True + + +class HTTPResetContent(HTTPSuccessful): + status_code = 205 + empty_body = True + + +class HTTPPartialContent(HTTPSuccessful): + status_code = 206 + + +############################################################ +# 3xx redirection +############################################################ + + +class HTTPMove(HTTPRedirection): + def __init__( + self, + location: StrOrURL, + *, + headers: Optional[LooseHeaders] = None, + reason: Optional[str] = None, + body: Any = None, + text: Optional[str] = None, + content_type: Optional[str] = None, + ) -> None: + if not location: + raise ValueError("HTTP redirects need a location to redirect to.") + super().__init__( + headers=headers, + reason=reason, + body=body, + text=text, + content_type=content_type, + ) + self.headers["Location"] = str(URL(location)) + self.location = location + + +class HTTPMultipleChoices(HTTPMove): + status_code = 300 + + +class HTTPMovedPermanently(HTTPMove): + status_code = 301 + + +class HTTPFound(HTTPMove): + status_code = 302 + + +# This one is safe after a POST (the redirected location will be +# retrieved with GET): +class HTTPSeeOther(HTTPMove): + status_code = 303 + + +class HTTPNotModified(HTTPRedirection): + # FIXME: this should include a date or etag header + status_code = 304 + empty_body = True + + +class HTTPUseProxy(HTTPMove): + # Not a move, but looks a little like one + status_code = 305 + + +class HTTPTemporaryRedirect(HTTPMove): + status_code = 307 + + +class HTTPPermanentRedirect(HTTPMove): + status_code = 308 + + +############################################################ +# 4xx client error +############################################################ + + +class HTTPClientError(HTTPError): + pass + + +class HTTPBadRequest(HTTPClientError): + status_code = 400 + + +class HTTPUnauthorized(HTTPClientError): + status_code = 401 + + +class HTTPPaymentRequired(HTTPClientError): + status_code = 402 + + +class HTTPForbidden(HTTPClientError): + status_code = 403 + + +class HTTPNotFound(HTTPClientError): + status_code = 404 + + +class HTTPMethodNotAllowed(HTTPClientError): + status_code = 405 + + def __init__( + self, + method: str, + allowed_methods: Iterable[str], + *, + headers: Optional[LooseHeaders] = None, + reason: Optional[str] = None, + body: Any = None, + text: Optional[str] = None, + content_type: Optional[str] = None, + ) -> None: + allow = ",".join(sorted(allowed_methods)) + super().__init__( + headers=headers, + reason=reason, + body=body, + text=text, + content_type=content_type, + ) + self.headers["Allow"] = allow + self.allowed_methods: Set[str] = set(allowed_methods) + self.method = method.upper() + + +class HTTPNotAcceptable(HTTPClientError): + status_code = 406 + + +class HTTPProxyAuthenticationRequired(HTTPClientError): + status_code = 407 + + +class HTTPRequestTimeout(HTTPClientError): + status_code = 408 + + +class HTTPConflict(HTTPClientError): + status_code = 409 + + +class HTTPGone(HTTPClientError): + status_code = 410 + + +class HTTPLengthRequired(HTTPClientError): + status_code = 411 + + +class HTTPPreconditionFailed(HTTPClientError): + status_code = 412 + + +class HTTPRequestEntityTooLarge(HTTPClientError): + status_code = 413 + + def __init__(self, max_size: float, actual_size: float, **kwargs: Any) -> None: + kwargs.setdefault( + "text", + "Maximum request body size {} exceeded, " + "actual body size {}".format(max_size, actual_size), + ) + super().__init__(**kwargs) + + +class HTTPRequestURITooLong(HTTPClientError): + status_code = 414 + + +class HTTPUnsupportedMediaType(HTTPClientError): + status_code = 415 + + +class HTTPRequestRangeNotSatisfiable(HTTPClientError): + status_code = 416 + + +class HTTPExpectationFailed(HTTPClientError): + status_code = 417 + + +class HTTPMisdirectedRequest(HTTPClientError): + status_code = 421 + + +class HTTPUnprocessableEntity(HTTPClientError): + status_code = 422 + + +class HTTPFailedDependency(HTTPClientError): + status_code = 424 + + +class HTTPUpgradeRequired(HTTPClientError): + status_code = 426 + + +class HTTPPreconditionRequired(HTTPClientError): + status_code = 428 + + +class HTTPTooManyRequests(HTTPClientError): + status_code = 429 + + +class HTTPRequestHeaderFieldsTooLarge(HTTPClientError): + status_code = 431 + + +class HTTPUnavailableForLegalReasons(HTTPClientError): + status_code = 451 + + def __init__( + self, + link: Optional[StrOrURL], + *, + headers: Optional[LooseHeaders] = None, + reason: Optional[str] = None, + body: Any = None, + text: Optional[str] = None, + content_type: Optional[str] = None, + ) -> None: + super().__init__( + headers=headers, + reason=reason, + body=body, + text=text, + content_type=content_type, + ) + self._link = None + if link: + self._link = URL(link) + self.headers["Link"] = f'<{str(self._link)}>; rel="blocked-by"' + + @property + def link(self) -> Optional[URL]: + return self._link + + +############################################################ +# 5xx Server Error +############################################################ +# Response status codes beginning with the digit "5" indicate cases in +# which the server is aware that it has erred or is incapable of +# performing the request. Except when responding to a HEAD request, the +# server SHOULD include an entity containing an explanation of the error +# situation, and whether it is a temporary or permanent condition. User +# agents SHOULD display any included entity to the user. These response +# codes are applicable to any request method. + + +class HTTPServerError(HTTPError): + pass + + +class HTTPInternalServerError(HTTPServerError): + status_code = 500 + + +class HTTPNotImplemented(HTTPServerError): + status_code = 501 + + +class HTTPBadGateway(HTTPServerError): + status_code = 502 + + +class HTTPServiceUnavailable(HTTPServerError): + status_code = 503 + + +class HTTPGatewayTimeout(HTTPServerError): + status_code = 504 + + +class HTTPVersionNotSupported(HTTPServerError): + status_code = 505 + + +class HTTPVariantAlsoNegotiates(HTTPServerError): + status_code = 506 + + +class HTTPInsufficientStorage(HTTPServerError): + status_code = 507 + + +class HTTPNotExtended(HTTPServerError): + status_code = 510 + + +class HTTPNetworkAuthenticationRequired(HTTPServerError): + status_code = 511 diff --git a/lib/python3.1/site-packages/aiohttp/web_fileresponse.py b/lib/python3.1/site-packages/aiohttp/web_fileresponse.py new file mode 100644 index 0000000000000000000000000000000000000000..344611cc4951b6ea63a49155c40ece8d3499b8fb --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web_fileresponse.py @@ -0,0 +1,418 @@ +import asyncio +import io +import os +import pathlib +import sys +from contextlib import suppress +from enum import Enum, auto +from mimetypes import MimeTypes +from stat import S_ISREG +from types import MappingProxyType +from typing import ( # noqa + IO, + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Final, + Iterator, + List, + Optional, + Set, + Tuple, + Union, + cast, +) + +from . import hdrs +from .abc import AbstractStreamWriter +from .helpers import ETAG_ANY, ETag, must_be_empty_body +from .typedefs import LooseHeaders, PathLike +from .web_exceptions import ( + HTTPForbidden, + HTTPNotFound, + HTTPNotModified, + HTTPPartialContent, + HTTPPreconditionFailed, + HTTPRequestRangeNotSatisfiable, +) +from .web_response import StreamResponse + +__all__ = ("FileResponse",) + +if TYPE_CHECKING: + from .web_request import BaseRequest + + +_T_OnChunkSent = Optional[Callable[[bytes], Awaitable[None]]] + + +NOSENDFILE: Final[bool] = bool(os.environ.get("AIOHTTP_NOSENDFILE")) + +CONTENT_TYPES: Final[MimeTypes] = MimeTypes() + +# File extension to IANA encodings map that will be checked in the order defined. +ENCODING_EXTENSIONS = MappingProxyType( + {ext: CONTENT_TYPES.encodings_map[ext] for ext in (".br", ".gz")} +) + +FALLBACK_CONTENT_TYPE = "application/octet-stream" + +# Provide additional MIME type/extension pairs to be recognized. +# https://en.wikipedia.org/wiki/List_of_archive_formats#Compression_only +ADDITIONAL_CONTENT_TYPES = MappingProxyType( + { + "application/gzip": ".gz", + "application/x-brotli": ".br", + "application/x-bzip2": ".bz2", + "application/x-compress": ".Z", + "application/x-xz": ".xz", + } +) + + +class _FileResponseResult(Enum): + """The result of the file response.""" + + SEND_FILE = auto() # Ie a regular file to send + NOT_ACCEPTABLE = auto() # Ie a socket, or non-regular file + PRE_CONDITION_FAILED = auto() # Ie If-Match or If-None-Match failed + NOT_MODIFIED = auto() # 304 Not Modified + + +# Add custom pairs and clear the encodings map so guess_type ignores them. +CONTENT_TYPES.encodings_map.clear() +for content_type, extension in ADDITIONAL_CONTENT_TYPES.items(): + CONTENT_TYPES.add_type(content_type, extension) + + +_CLOSE_FUTURES: Set[asyncio.Future[None]] = set() + + +class FileResponse(StreamResponse): + """A response object can be used to send files.""" + + def __init__( + self, + path: PathLike, + chunk_size: int = 256 * 1024, + status: int = 200, + reason: Optional[str] = None, + headers: Optional[LooseHeaders] = None, + ) -> None: + super().__init__(status=status, reason=reason, headers=headers) + + self._path = pathlib.Path(path) + self._chunk_size = chunk_size + + def _seek_and_read(self, fobj: IO[Any], offset: int, chunk_size: int) -> bytes: + fobj.seek(offset) + return fobj.read(chunk_size) # type: ignore[no-any-return] + + async def _sendfile_fallback( + self, writer: AbstractStreamWriter, fobj: IO[Any], offset: int, count: int + ) -> AbstractStreamWriter: + # To keep memory usage low,fobj is transferred in chunks + # controlled by the constructor's chunk_size argument. + + chunk_size = self._chunk_size + loop = asyncio.get_event_loop() + chunk = await loop.run_in_executor( + None, self._seek_and_read, fobj, offset, chunk_size + ) + while chunk: + await writer.write(chunk) + count = count - chunk_size + if count <= 0: + break + chunk = await loop.run_in_executor(None, fobj.read, min(chunk_size, count)) + + await writer.drain() + return writer + + async def _sendfile( + self, request: "BaseRequest", fobj: IO[Any], offset: int, count: int + ) -> AbstractStreamWriter: + writer = await super().prepare(request) + assert writer is not None + + if NOSENDFILE or self.compression: + return await self._sendfile_fallback(writer, fobj, offset, count) + + loop = request._loop + transport = request.transport + assert transport is not None + + try: + await loop.sendfile(transport, fobj, offset, count) + except NotImplementedError: + return await self._sendfile_fallback(writer, fobj, offset, count) + + await super().write_eof() + return writer + + @staticmethod + def _etag_match(etag_value: str, etags: Tuple[ETag, ...], *, weak: bool) -> bool: + if len(etags) == 1 and etags[0].value == ETAG_ANY: + return True + return any( + etag.value == etag_value for etag in etags if weak or not etag.is_weak + ) + + async def _not_modified( + self, request: "BaseRequest", etag_value: str, last_modified: float + ) -> Optional[AbstractStreamWriter]: + self.set_status(HTTPNotModified.status_code) + self._length_check = False + self.etag = etag_value # type: ignore[assignment] + self.last_modified = last_modified # type: ignore[assignment] + # Delete any Content-Length headers provided by user. HTTP 304 + # should always have empty response body + return await super().prepare(request) + + async def _precondition_failed( + self, request: "BaseRequest" + ) -> Optional[AbstractStreamWriter]: + self.set_status(HTTPPreconditionFailed.status_code) + self.content_length = 0 + return await super().prepare(request) + + def _make_response( + self, request: "BaseRequest", accept_encoding: str + ) -> Tuple[ + _FileResponseResult, Optional[io.BufferedReader], os.stat_result, Optional[str] + ]: + """Return the response result, io object, stat result, and encoding. + + If an uncompressed file is returned, the encoding is set to + :py:data:`None`. + + This method should be called from a thread executor + since it calls os.stat which may block. + """ + file_path, st, file_encoding = self._get_file_path_stat_encoding( + accept_encoding + ) + if not file_path: + return _FileResponseResult.NOT_ACCEPTABLE, None, st, None + + etag_value = f"{st.st_mtime_ns:x}-{st.st_size:x}" + + # https://www.rfc-editor.org/rfc/rfc9110#section-13.1.1-2 + if (ifmatch := request.if_match) is not None and not self._etag_match( + etag_value, ifmatch, weak=False + ): + return _FileResponseResult.PRE_CONDITION_FAILED, None, st, file_encoding + + if ( + (unmodsince := request.if_unmodified_since) is not None + and ifmatch is None + and st.st_mtime > unmodsince.timestamp() + ): + return _FileResponseResult.PRE_CONDITION_FAILED, None, st, file_encoding + + # https://www.rfc-editor.org/rfc/rfc9110#section-13.1.2-2 + if (ifnonematch := request.if_none_match) is not None and self._etag_match( + etag_value, ifnonematch, weak=True + ): + return _FileResponseResult.NOT_MODIFIED, None, st, file_encoding + + if ( + (modsince := request.if_modified_since) is not None + and ifnonematch is None + and st.st_mtime <= modsince.timestamp() + ): + return _FileResponseResult.NOT_MODIFIED, None, st, file_encoding + + fobj = file_path.open("rb") + with suppress(OSError): + # fstat() may not be available on all platforms + # Once we open the file, we want the fstat() to ensure + # the file has not changed between the first stat() + # and the open(). + st = os.stat(fobj.fileno()) + return _FileResponseResult.SEND_FILE, fobj, st, file_encoding + + def _get_file_path_stat_encoding( + self, accept_encoding: str + ) -> Tuple[Optional[pathlib.Path], os.stat_result, Optional[str]]: + file_path = self._path + for file_extension, file_encoding in ENCODING_EXTENSIONS.items(): + if file_encoding not in accept_encoding: + continue + + compressed_path = file_path.with_suffix(file_path.suffix + file_extension) + with suppress(OSError): + # Do not follow symlinks and ignore any non-regular files. + st = compressed_path.lstat() + if S_ISREG(st.st_mode): + return compressed_path, st, file_encoding + + # Fallback to the uncompressed file + st = file_path.stat() + return file_path if S_ISREG(st.st_mode) else None, st, None + + async def prepare(self, request: "BaseRequest") -> Optional[AbstractStreamWriter]: + loop = asyncio.get_running_loop() + # Encoding comparisons should be case-insensitive + # https://www.rfc-editor.org/rfc/rfc9110#section-8.4.1 + accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower() + try: + response_result, fobj, st, file_encoding = await loop.run_in_executor( + None, self._make_response, request, accept_encoding + ) + except PermissionError: + self.set_status(HTTPForbidden.status_code) + return await super().prepare(request) + except OSError: + # Most likely to be FileNotFoundError or OSError for circular + # symlinks in python >= 3.13, so respond with 404. + self.set_status(HTTPNotFound.status_code) + return await super().prepare(request) + + # Forbid special files like sockets, pipes, devices, etc. + if response_result is _FileResponseResult.NOT_ACCEPTABLE: + self.set_status(HTTPForbidden.status_code) + return await super().prepare(request) + + if response_result is _FileResponseResult.PRE_CONDITION_FAILED: + return await self._precondition_failed(request) + + if response_result is _FileResponseResult.NOT_MODIFIED: + etag_value = f"{st.st_mtime_ns:x}-{st.st_size:x}" + last_modified = st.st_mtime + return await self._not_modified(request, etag_value, last_modified) + + assert fobj is not None + try: + return await self._prepare_open_file(request, fobj, st, file_encoding) + finally: + # We do not await here because we do not want to wait + # for the executor to finish before returning the response + # so the connection can begin servicing another request + # as soon as possible. + close_future = loop.run_in_executor(None, fobj.close) + # Hold a strong reference to the future to prevent it from being + # garbage collected before it completes. + _CLOSE_FUTURES.add(close_future) + close_future.add_done_callback(_CLOSE_FUTURES.remove) + + async def _prepare_open_file( + self, + request: "BaseRequest", + fobj: io.BufferedReader, + st: os.stat_result, + file_encoding: Optional[str], + ) -> Optional[AbstractStreamWriter]: + status = self._status + file_size: int = st.st_size + file_mtime: float = st.st_mtime + count: int = file_size + start: Optional[int] = None + + if (ifrange := request.if_range) is None or file_mtime <= ifrange.timestamp(): + # If-Range header check: + # condition = cached date >= last modification date + # return 206 if True else 200. + # if False: + # Range header would not be processed, return 200 + # if True but Range header missing + # return 200 + try: + rng = request.http_range + start = rng.start + end: Optional[int] = rng.stop + except ValueError: + # https://tools.ietf.org/html/rfc7233: + # A server generating a 416 (Range Not Satisfiable) response to + # a byte-range request SHOULD send a Content-Range header field + # with an unsatisfied-range value. + # The complete-length in a 416 response indicates the current + # length of the selected representation. + # + # Will do the same below. Many servers ignore this and do not + # send a Content-Range header with HTTP 416 + self._headers[hdrs.CONTENT_RANGE] = f"bytes */{file_size}" + self.set_status(HTTPRequestRangeNotSatisfiable.status_code) + return await super().prepare(request) + + # If a range request has been made, convert start, end slice + # notation into file pointer offset and count + if start is not None: + if start < 0 and end is None: # return tail of file + start += file_size + if start < 0: + # if Range:bytes=-1000 in request header but file size + # is only 200, there would be trouble without this + start = 0 + count = file_size - start + else: + # rfc7233:If the last-byte-pos value is + # absent, or if the value is greater than or equal to + # the current length of the representation data, + # the byte range is interpreted as the remainder + # of the representation (i.e., the server replaces the + # value of last-byte-pos with a value that is one less than + # the current length of the selected representation). + count = ( + min(end if end is not None else file_size, file_size) - start + ) + + if start >= file_size: + # HTTP 416 should be returned in this case. + # + # According to https://tools.ietf.org/html/rfc7233: + # If a valid byte-range-set includes at least one + # byte-range-spec with a first-byte-pos that is less than + # the current length of the representation, or at least one + # suffix-byte-range-spec with a non-zero suffix-length, + # then the byte-range-set is satisfiable. Otherwise, the + # byte-range-set is unsatisfiable. + self._headers[hdrs.CONTENT_RANGE] = f"bytes */{file_size}" + self.set_status(HTTPRequestRangeNotSatisfiable.status_code) + return await super().prepare(request) + + status = HTTPPartialContent.status_code + # Even though you are sending the whole file, you should still + # return a HTTP 206 for a Range request. + self.set_status(status) + + # If the Content-Type header is not already set, guess it based on the + # extension of the request path. The encoding returned by guess_type + # can be ignored since the map was cleared above. + if hdrs.CONTENT_TYPE not in self._headers: + if sys.version_info >= (3, 13): + guesser = CONTENT_TYPES.guess_file_type + else: + guesser = CONTENT_TYPES.guess_type + self.content_type = guesser(self._path)[0] or FALLBACK_CONTENT_TYPE + + if file_encoding: + self._headers[hdrs.CONTENT_ENCODING] = file_encoding + self._headers[hdrs.VARY] = hdrs.ACCEPT_ENCODING + # Disable compression if we are already sending + # a compressed file since we don't want to double + # compress. + self._compression = False + + self.etag = f"{st.st_mtime_ns:x}-{st.st_size:x}" # type: ignore[assignment] + self.last_modified = file_mtime # type: ignore[assignment] + self.content_length = count + + self._headers[hdrs.ACCEPT_RANGES] = "bytes" + + if status == HTTPPartialContent.status_code: + real_start = start + assert real_start is not None + self._headers[hdrs.CONTENT_RANGE] = "bytes {}-{}/{}".format( + real_start, real_start + count - 1, file_size + ) + + # If we are sending 0 bytes calling sendfile() will throw a ValueError + if count == 0 or must_be_empty_body(request.method, status): + return await super().prepare(request) + + # be aware that start could be None or int=0 here. + offset = start or 0 + + return await self._sendfile(request, fobj, offset, count) diff --git a/lib/python3.1/site-packages/aiohttp/web_log.py b/lib/python3.1/site-packages/aiohttp/web_log.py new file mode 100644 index 0000000000000000000000000000000000000000..d5ea2beeb152974ce5dd9f3e7990133ce04f7980 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web_log.py @@ -0,0 +1,216 @@ +import datetime +import functools +import logging +import os +import re +import time as time_mod +from collections import namedtuple +from typing import Any, Callable, Dict, Iterable, List, Tuple # noqa + +from .abc import AbstractAccessLogger +from .web_request import BaseRequest +from .web_response import StreamResponse + +KeyMethod = namedtuple("KeyMethod", "key method") + + +class AccessLogger(AbstractAccessLogger): + """Helper object to log access. + + Usage: + log = logging.getLogger("spam") + log_format = "%a %{User-Agent}i" + access_logger = AccessLogger(log, log_format) + access_logger.log(request, response, time) + + Format: + %% The percent sign + %a Remote IP-address (IP-address of proxy if using reverse proxy) + %t Time when the request was started to process + %P The process ID of the child that serviced the request + %r First line of request + %s Response status code + %b Size of response in bytes, including HTTP headers + %T Time taken to serve the request, in seconds + %Tf Time taken to serve the request, in seconds with floating fraction + in .06f format + %D Time taken to serve the request, in microseconds + %{FOO}i request.headers['FOO'] + %{FOO}o response.headers['FOO'] + %{FOO}e os.environ['FOO'] + + """ + + LOG_FORMAT_MAP = { + "a": "remote_address", + "t": "request_start_time", + "P": "process_id", + "r": "first_request_line", + "s": "response_status", + "b": "response_size", + "T": "request_time", + "Tf": "request_time_frac", + "D": "request_time_micro", + "i": "request_header", + "o": "response_header", + } + + LOG_FORMAT = '%a %t "%r" %s %b "%{Referer}i" "%{User-Agent}i"' + FORMAT_RE = re.compile(r"%(\{([A-Za-z0-9\-_]+)\}([ioe])|[atPrsbOD]|Tf?)") + CLEANUP_RE = re.compile(r"(%[^s])") + _FORMAT_CACHE: Dict[str, Tuple[str, List[KeyMethod]]] = {} + + def __init__(self, logger: logging.Logger, log_format: str = LOG_FORMAT) -> None: + """Initialise the logger. + + logger is a logger object to be used for logging. + log_format is a string with apache compatible log format description. + + """ + super().__init__(logger, log_format=log_format) + + _compiled_format = AccessLogger._FORMAT_CACHE.get(log_format) + if not _compiled_format: + _compiled_format = self.compile_format(log_format) + AccessLogger._FORMAT_CACHE[log_format] = _compiled_format + + self._log_format, self._methods = _compiled_format + + def compile_format(self, log_format: str) -> Tuple[str, List[KeyMethod]]: + """Translate log_format into form usable by modulo formatting + + All known atoms will be replaced with %s + Also methods for formatting of those atoms will be added to + _methods in appropriate order + + For example we have log_format = "%a %t" + This format will be translated to "%s %s" + Also contents of _methods will be + [self._format_a, self._format_t] + These method will be called and results will be passed + to translated string format. + + Each _format_* method receive 'args' which is list of arguments + given to self.log + + Exceptions are _format_e, _format_i and _format_o methods which + also receive key name (by functools.partial) + + """ + # list of (key, method) tuples, we don't use an OrderedDict as users + # can repeat the same key more than once + methods = list() + + for atom in self.FORMAT_RE.findall(log_format): + if atom[1] == "": + format_key1 = self.LOG_FORMAT_MAP[atom[0]] + m = getattr(AccessLogger, "_format_%s" % atom[0]) + key_method = KeyMethod(format_key1, m) + else: + format_key2 = (self.LOG_FORMAT_MAP[atom[2]], atom[1]) + m = getattr(AccessLogger, "_format_%s" % atom[2]) + key_method = KeyMethod(format_key2, functools.partial(m, atom[1])) + + methods.append(key_method) + + log_format = self.FORMAT_RE.sub(r"%s", log_format) + log_format = self.CLEANUP_RE.sub(r"%\1", log_format) + return log_format, methods + + @staticmethod + def _format_i( + key: str, request: BaseRequest, response: StreamResponse, time: float + ) -> str: + if request is None: + return "(no headers)" + + # suboptimal, make istr(key) once + return request.headers.get(key, "-") + + @staticmethod + def _format_o( + key: str, request: BaseRequest, response: StreamResponse, time: float + ) -> str: + # suboptimal, make istr(key) once + return response.headers.get(key, "-") + + @staticmethod + def _format_a(request: BaseRequest, response: StreamResponse, time: float) -> str: + if request is None: + return "-" + ip = request.remote + return ip if ip is not None else "-" + + @staticmethod + def _format_t(request: BaseRequest, response: StreamResponse, time: float) -> str: + tz = datetime.timezone(datetime.timedelta(seconds=-time_mod.timezone)) + now = datetime.datetime.now(tz) + start_time = now - datetime.timedelta(seconds=time) + return start_time.strftime("[%d/%b/%Y:%H:%M:%S %z]") + + @staticmethod + def _format_P(request: BaseRequest, response: StreamResponse, time: float) -> str: + return "<%s>" % os.getpid() + + @staticmethod + def _format_r(request: BaseRequest, response: StreamResponse, time: float) -> str: + if request is None: + return "-" + return "{} {} HTTP/{}.{}".format( + request.method, + request.path_qs, + request.version.major, + request.version.minor, + ) + + @staticmethod + def _format_s(request: BaseRequest, response: StreamResponse, time: float) -> int: + return response.status + + @staticmethod + def _format_b(request: BaseRequest, response: StreamResponse, time: float) -> int: + return response.body_length + + @staticmethod + def _format_T(request: BaseRequest, response: StreamResponse, time: float) -> str: + return str(round(time)) + + @staticmethod + def _format_Tf(request: BaseRequest, response: StreamResponse, time: float) -> str: + return "%06f" % time + + @staticmethod + def _format_D(request: BaseRequest, response: StreamResponse, time: float) -> str: + return str(round(time * 1000000)) + + def _format_line( + self, request: BaseRequest, response: StreamResponse, time: float + ) -> Iterable[Tuple[str, Callable[[BaseRequest, StreamResponse, float], str]]]: + return [(key, method(request, response, time)) for key, method in self._methods] + + @property + def enabled(self) -> bool: + """Check if logger is enabled.""" + # Avoid formatting the log line if it will not be emitted. + return self.logger.isEnabledFor(logging.INFO) + + def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None: + try: + fmt_info = self._format_line(request, response, time) + + values = list() + extra = dict() + for key, value in fmt_info: + values.append(value) + + if key.__class__ is str: + extra[key] = value + else: + k1, k2 = key # type: ignore[misc] + dct = extra.get(k1, {}) # type: ignore[var-annotated,has-type] + dct[k2] = value # type: ignore[index,has-type] + extra[k1] = dct # type: ignore[has-type,assignment] + + self.logger.info(self._log_format % tuple(values), extra=extra) + except Exception: + self.logger.exception("Error in logging") diff --git a/lib/python3.1/site-packages/aiohttp/web_middlewares.py b/lib/python3.1/site-packages/aiohttp/web_middlewares.py new file mode 100644 index 0000000000000000000000000000000000000000..2f1f5f58e6e38845d4d2d4ffdd2748fc519fa5bf --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web_middlewares.py @@ -0,0 +1,121 @@ +import re +from typing import TYPE_CHECKING, Tuple, Type, TypeVar + +from .typedefs import Handler, Middleware +from .web_exceptions import HTTPMove, HTTPPermanentRedirect +from .web_request import Request +from .web_response import StreamResponse +from .web_urldispatcher import SystemRoute + +__all__ = ( + "middleware", + "normalize_path_middleware", +) + +if TYPE_CHECKING: + from .web_app import Application + +_Func = TypeVar("_Func") + + +async def _check_request_resolves(request: Request, path: str) -> Tuple[bool, Request]: + alt_request = request.clone(rel_url=path) + + match_info = await request.app.router.resolve(alt_request) + alt_request._match_info = match_info + + if match_info.http_exception is None: + return True, alt_request + + return False, request + + +def middleware(f: _Func) -> _Func: + f.__middleware_version__ = 1 # type: ignore[attr-defined] + return f + + +def normalize_path_middleware( + *, + append_slash: bool = True, + remove_slash: bool = False, + merge_slashes: bool = True, + redirect_class: Type[HTTPMove] = HTTPPermanentRedirect, +) -> Middleware: + """Factory for producing a middleware that normalizes the path of a request. + + Normalizing means: + - Add or remove a trailing slash to the path. + - Double slashes are replaced by one. + + The middleware returns as soon as it finds a path that resolves + correctly. The order if both merge and append/remove are enabled is + 1) merge slashes + 2) append/remove slash + 3) both merge slashes and append/remove slash. + If the path resolves with at least one of those conditions, it will + redirect to the new path. + + Only one of `append_slash` and `remove_slash` can be enabled. If both + are `True` the factory will raise an assertion error + + If `append_slash` is `True` the middleware will append a slash when + needed. If a resource is defined with trailing slash and the request + comes without it, it will append it automatically. + + If `remove_slash` is `True`, `append_slash` must be `False`. When enabled + the middleware will remove trailing slashes and redirect if the resource + is defined + + If merge_slashes is True, merge multiple consecutive slashes in the + path into one. + """ + correct_configuration = not (append_slash and remove_slash) + assert correct_configuration, "Cannot both remove and append slash" + + @middleware + async def impl(request: Request, handler: Handler) -> StreamResponse: + if isinstance(request.match_info.route, SystemRoute): + paths_to_check = [] + if "?" in request.raw_path: + path, query = request.raw_path.split("?", 1) + query = "?" + query + else: + query = "" + path = request.raw_path + + if merge_slashes: + paths_to_check.append(re.sub("//+", "/", path)) + if append_slash and not request.path.endswith("/"): + paths_to_check.append(path + "/") + if remove_slash and request.path.endswith("/"): + paths_to_check.append(path[:-1]) + if merge_slashes and append_slash: + paths_to_check.append(re.sub("//+", "/", path + "/")) + if merge_slashes and remove_slash: + merged_slashes = re.sub("//+", "/", path) + paths_to_check.append(merged_slashes[:-1]) + + for path in paths_to_check: + path = re.sub("^//+", "/", path) # SECURITY: GHSA-v6wp-4m6f-gcjg + resolves, request = await _check_request_resolves(request, path) + if resolves: + raise redirect_class(request.raw_path + query) + + return await handler(request) + + return impl + + +def _fix_request_current_app(app: "Application") -> Middleware: + @middleware + async def impl(request: Request, handler: Handler) -> StreamResponse: + match_info = request.match_info + prev = match_info.current_app + match_info.current_app = app + try: + return await handler(request) + finally: + match_info.current_app = prev + + return impl diff --git a/lib/python3.1/site-packages/aiohttp/web_protocol.py b/lib/python3.1/site-packages/aiohttp/web_protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..e1923aac24bdc7f2c7697b061a6bf68ae6509c6f --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web_protocol.py @@ -0,0 +1,792 @@ +import asyncio +import asyncio.streams +import sys +import traceback +import warnings +from collections import deque +from contextlib import suppress +from html import escape as html_escape +from http import HTTPStatus +from logging import Logger +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Deque, + Optional, + Sequence, + Tuple, + Type, + Union, + cast, +) + +import attr +import yarl +from propcache import under_cached_property + +from .abc import AbstractAccessLogger, AbstractStreamWriter +from .base_protocol import BaseProtocol +from .helpers import ceil_timeout +from .http import ( + HttpProcessingError, + HttpRequestParser, + HttpVersion10, + RawRequestMessage, + StreamWriter, +) +from .http_exceptions import BadHttpMethod +from .log import access_logger, server_logger +from .streams import EMPTY_PAYLOAD, StreamReader +from .tcp_helpers import tcp_keepalive +from .web_exceptions import HTTPException, HTTPInternalServerError +from .web_log import AccessLogger +from .web_request import BaseRequest +from .web_response import Response, StreamResponse + +__all__ = ("RequestHandler", "RequestPayloadError", "PayloadAccessError") + +if TYPE_CHECKING: + import ssl + + from .web_server import Server + + +_RequestFactory = Callable[ + [ + RawRequestMessage, + StreamReader, + "RequestHandler", + AbstractStreamWriter, + "asyncio.Task[None]", + ], + BaseRequest, +] + +_RequestHandler = Callable[[BaseRequest], Awaitable[StreamResponse]] + +ERROR = RawRequestMessage( + "UNKNOWN", + "/", + HttpVersion10, + {}, # type: ignore[arg-type] + {}, # type: ignore[arg-type] + True, + None, + False, + False, + yarl.URL("/"), +) + + +class RequestPayloadError(Exception): + """Payload parsing error.""" + + +class PayloadAccessError(Exception): + """Payload was accessed after response was sent.""" + + +_PAYLOAD_ACCESS_ERROR = PayloadAccessError() + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class _ErrInfo: + status: int + exc: BaseException + message: str + + +_MsgType = Tuple[Union[RawRequestMessage, _ErrInfo], StreamReader] + + +class RequestHandler(BaseProtocol): + """HTTP protocol implementation. + + RequestHandler handles incoming HTTP request. It reads request line, + request headers and request payload and calls handle_request() method. + By default it always returns with 404 response. + + RequestHandler handles errors in incoming request, like bad + status line, bad headers or incomplete payload. If any error occurs, + connection gets closed. + + keepalive_timeout -- number of seconds before closing + keep-alive connection + + tcp_keepalive -- TCP keep-alive is on, default is on + + debug -- enable debug mode + + logger -- custom logger object + + access_log_class -- custom class for access_logger + + access_log -- custom logging object + + access_log_format -- access log format string + + loop -- Optional event loop + + max_line_size -- Optional maximum header line size + + max_field_size -- Optional maximum header field size + + max_headers -- Optional maximum header size + + timeout_ceil_threshold -- Optional value to specify + threshold to ceil() timeout + values + + """ + + __slots__ = ( + "_request_count", + "_keepalive", + "_manager", + "_request_handler", + "_request_factory", + "_tcp_keepalive", + "_next_keepalive_close_time", + "_keepalive_handle", + "_keepalive_timeout", + "_lingering_time", + "_messages", + "_message_tail", + "_handler_waiter", + "_waiter", + "_task_handler", + "_upgrade", + "_payload_parser", + "_request_parser", + "_reading_paused", + "logger", + "debug", + "access_log", + "access_logger", + "_close", + "_force_close", + "_current_request", + "_timeout_ceil_threshold", + "_request_in_progress", + "_logging_enabled", + "_cache", + ) + + def __init__( + self, + manager: "Server", + *, + loop: asyncio.AbstractEventLoop, + # Default should be high enough that it's likely longer than a reverse proxy. + keepalive_timeout: float = 3630, + tcp_keepalive: bool = True, + logger: Logger = server_logger, + access_log_class: Type[AbstractAccessLogger] = AccessLogger, + access_log: Logger = access_logger, + access_log_format: str = AccessLogger.LOG_FORMAT, + debug: bool = False, + max_line_size: int = 8190, + max_headers: int = 32768, + max_field_size: int = 8190, + lingering_time: float = 10.0, + read_bufsize: int = 2**16, + auto_decompress: bool = True, + timeout_ceil_threshold: float = 5, + ): + super().__init__(loop) + + # _request_count is the number of requests processed with the same connection. + self._request_count = 0 + self._keepalive = False + self._current_request: Optional[BaseRequest] = None + self._manager: Optional[Server] = manager + self._request_handler: Optional[_RequestHandler] = manager.request_handler + self._request_factory: Optional[_RequestFactory] = manager.request_factory + + self._tcp_keepalive = tcp_keepalive + # placeholder to be replaced on keepalive timeout setup + self._next_keepalive_close_time = 0.0 + self._keepalive_handle: Optional[asyncio.Handle] = None + self._keepalive_timeout = keepalive_timeout + self._lingering_time = float(lingering_time) + + self._messages: Deque[_MsgType] = deque() + self._message_tail = b"" + + self._waiter: Optional[asyncio.Future[None]] = None + self._handler_waiter: Optional[asyncio.Future[None]] = None + self._task_handler: Optional[asyncio.Task[None]] = None + + self._upgrade = False + self._payload_parser: Any = None + self._request_parser: Optional[HttpRequestParser] = HttpRequestParser( + self, + loop, + read_bufsize, + max_line_size=max_line_size, + max_field_size=max_field_size, + max_headers=max_headers, + payload_exception=RequestPayloadError, + auto_decompress=auto_decompress, + ) + + self._timeout_ceil_threshold: float = 5 + try: + self._timeout_ceil_threshold = float(timeout_ceil_threshold) + except (TypeError, ValueError): + pass + + self.logger = logger + self.debug = debug + self.access_log = access_log + if access_log: + self.access_logger: Optional[AbstractAccessLogger] = access_log_class( + access_log, access_log_format + ) + self._logging_enabled = self.access_logger.enabled + else: + self.access_logger = None + self._logging_enabled = False + + self._close = False + self._force_close = False + self._request_in_progress = False + self._cache: dict[str, Any] = {} + + def __repr__(self) -> str: + return "<{} {}>".format( + self.__class__.__name__, + "connected" if self.transport is not None else "disconnected", + ) + + @under_cached_property + def ssl_context(self) -> Optional["ssl.SSLContext"]: + """Return SSLContext if available.""" + return ( + None + if self.transport is None + else self.transport.get_extra_info("sslcontext") + ) + + @under_cached_property + def peername( + self, + ) -> Optional[Union[str, Tuple[str, int, int, int], Tuple[str, int]]]: + """Return peername if available.""" + return ( + None + if self.transport is None + else self.transport.get_extra_info("peername") + ) + + @property + def keepalive_timeout(self) -> float: + return self._keepalive_timeout + + async def shutdown(self, timeout: Optional[float] = 15.0) -> None: + """Do worker process exit preparations. + + We need to clean up everything and stop accepting requests. + It is especially important for keep-alive connections. + """ + self._force_close = True + + if self._keepalive_handle is not None: + self._keepalive_handle.cancel() + + # Wait for graceful handler completion + if self._request_in_progress: + # The future is only created when we are shutting + # down while the handler is still processing a request + # to avoid creating a future for every request. + self._handler_waiter = self._loop.create_future() + try: + async with ceil_timeout(timeout): + await self._handler_waiter + except (asyncio.CancelledError, asyncio.TimeoutError): + self._handler_waiter = None + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + # Then cancel handler and wait + try: + async with ceil_timeout(timeout): + if self._current_request is not None: + self._current_request._cancel(asyncio.CancelledError()) + + if self._task_handler is not None and not self._task_handler.done(): + await asyncio.shield(self._task_handler) + except (asyncio.CancelledError, asyncio.TimeoutError): + if ( + sys.version_info >= (3, 11) + and (task := asyncio.current_task()) + and task.cancelling() + ): + raise + + # force-close non-idle handler + if self._task_handler is not None: + self._task_handler.cancel() + + self.force_close() + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + super().connection_made(transport) + + real_transport = cast(asyncio.Transport, transport) + if self._tcp_keepalive: + tcp_keepalive(real_transport) + + assert self._manager is not None + self._manager.connection_made(self, real_transport) + + loop = self._loop + if sys.version_info >= (3, 12): + task = asyncio.Task(self.start(), loop=loop, eager_start=True) + else: + task = loop.create_task(self.start()) + self._task_handler = task + + def connection_lost(self, exc: Optional[BaseException]) -> None: + if self._manager is None: + return + self._manager.connection_lost(self, exc) + + # Grab value before setting _manager to None. + handler_cancellation = self._manager.handler_cancellation + + self.force_close() + super().connection_lost(exc) + self._manager = None + self._request_factory = None + self._request_handler = None + self._request_parser = None + + if self._keepalive_handle is not None: + self._keepalive_handle.cancel() + + if self._current_request is not None: + if exc is None: + exc = ConnectionResetError("Connection lost") + self._current_request._cancel(exc) + + if handler_cancellation and self._task_handler is not None: + self._task_handler.cancel() + + self._task_handler = None + + if self._payload_parser is not None: + self._payload_parser.feed_eof() + self._payload_parser = None + + def set_parser(self, parser: Any) -> None: + # Actual type is WebReader + assert self._payload_parser is None + + self._payload_parser = parser + + if self._message_tail: + self._payload_parser.feed_data(self._message_tail) + self._message_tail = b"" + + def eof_received(self) -> None: + pass + + def data_received(self, data: bytes) -> None: + if self._force_close or self._close: + return + # parse http messages + messages: Sequence[_MsgType] + if self._payload_parser is None and not self._upgrade: + assert self._request_parser is not None + try: + messages, upgraded, tail = self._request_parser.feed_data(data) + except HttpProcessingError as exc: + messages = [ + (_ErrInfo(status=400, exc=exc, message=exc.message), EMPTY_PAYLOAD) + ] + upgraded = False + tail = b"" + + for msg, payload in messages or (): + self._request_count += 1 + self._messages.append((msg, payload)) + + waiter = self._waiter + if messages and waiter is not None and not waiter.done(): + # don't set result twice + waiter.set_result(None) + + self._upgrade = upgraded + if upgraded and tail: + self._message_tail = tail + + # no parser, just store + elif self._payload_parser is None and self._upgrade and data: + self._message_tail += data + + # feed payload + elif data: + eof, tail = self._payload_parser.feed_data(data) + if eof: + self.close() + + def keep_alive(self, val: bool) -> None: + """Set keep-alive connection mode. + + :param bool val: new state. + """ + self._keepalive = val + if self._keepalive_handle: + self._keepalive_handle.cancel() + self._keepalive_handle = None + + def close(self) -> None: + """Close connection. + + Stop accepting new pipelining messages and close + connection when handlers done processing messages. + """ + self._close = True + if self._waiter: + self._waiter.cancel() + + def force_close(self) -> None: + """Forcefully close connection.""" + self._force_close = True + if self._waiter: + self._waiter.cancel() + if self.transport is not None: + self.transport.close() + self.transport = None + + def log_access( + self, request: BaseRequest, response: StreamResponse, time: Optional[float] + ) -> None: + if self.access_logger is not None and self.access_logger.enabled: + if TYPE_CHECKING: + assert time is not None + self.access_logger.log(request, response, self._loop.time() - time) + + def log_debug(self, *args: Any, **kw: Any) -> None: + if self.debug: + self.logger.debug(*args, **kw) + + def log_exception(self, *args: Any, **kw: Any) -> None: + self.logger.exception(*args, **kw) + + def _process_keepalive(self) -> None: + self._keepalive_handle = None + if self._force_close or not self._keepalive: + return + + loop = self._loop + now = loop.time() + close_time = self._next_keepalive_close_time + if now < close_time: + # Keep alive close check fired too early, reschedule + self._keepalive_handle = loop.call_at(close_time, self._process_keepalive) + return + + # handler in idle state + if self._waiter and not self._waiter.done(): + self.force_close() + + async def _handle_request( + self, + request: BaseRequest, + start_time: Optional[float], + request_handler: Callable[[BaseRequest], Awaitable[StreamResponse]], + ) -> Tuple[StreamResponse, bool]: + self._request_in_progress = True + try: + try: + self._current_request = request + resp = await request_handler(request) + finally: + self._current_request = None + except HTTPException as exc: + resp = exc + resp, reset = await self.finish_response(request, resp, start_time) + except asyncio.CancelledError: + raise + except asyncio.TimeoutError as exc: + self.log_debug("Request handler timed out.", exc_info=exc) + resp = self.handle_error(request, 504) + resp, reset = await self.finish_response(request, resp, start_time) + except Exception as exc: + resp = self.handle_error(request, 500, exc) + resp, reset = await self.finish_response(request, resp, start_time) + else: + # Deprecation warning (See #2415) + if getattr(resp, "__http_exception__", False): + warnings.warn( + "returning HTTPException object is deprecated " + "(#2415) and will be removed, " + "please raise the exception instead", + DeprecationWarning, + ) + + resp, reset = await self.finish_response(request, resp, start_time) + finally: + self._request_in_progress = False + if self._handler_waiter is not None: + self._handler_waiter.set_result(None) + + return resp, reset + + async def start(self) -> None: + """Process incoming request. + + It reads request line, request headers and request payload, then + calls handle_request() method. Subclass has to override + handle_request(). start() handles various exceptions in request + or response handling. Connection is being closed always unless + keep_alive(True) specified. + """ + loop = self._loop + manager = self._manager + assert manager is not None + keepalive_timeout = self._keepalive_timeout + resp = None + assert self._request_factory is not None + assert self._request_handler is not None + + while not self._force_close: + if not self._messages: + try: + # wait for next request + self._waiter = loop.create_future() + await self._waiter + finally: + self._waiter = None + + message, payload = self._messages.popleft() + + # time is only fetched if logging is enabled as otherwise + # its thrown away and never used. + start = loop.time() if self._logging_enabled else None + + manager.requests_count += 1 + writer = StreamWriter(self, loop) + if isinstance(message, _ErrInfo): + # make request_factory work + request_handler = self._make_error_handler(message) + message = ERROR + else: + request_handler = self._request_handler + + # Important don't hold a reference to the current task + # as on traceback it will prevent the task from being + # collected and will cause a memory leak. + request = self._request_factory( + message, + payload, + self, + writer, + self._task_handler or asyncio.current_task(loop), # type: ignore[arg-type] + ) + try: + # a new task is used for copy context vars (#3406) + coro = self._handle_request(request, start, request_handler) + if sys.version_info >= (3, 12): + task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + task = loop.create_task(coro) + try: + resp, reset = await task + except ConnectionError: + self.log_debug("Ignored premature client disconnection") + break + + # Drop the processed task from asyncio.Task.all_tasks() early + del task + if reset: + self.log_debug("Ignored premature client disconnection 2") + break + + # notify server about keep-alive + self._keepalive = bool(resp.keep_alive) + + # check payload + if not payload.is_eof(): + lingering_time = self._lingering_time + if not self._force_close and lingering_time: + self.log_debug( + "Start lingering close timer for %s sec.", lingering_time + ) + + now = loop.time() + end_t = now + lingering_time + + try: + while not payload.is_eof() and now < end_t: + async with ceil_timeout(end_t - now): + # read and ignore + await payload.readany() + now = loop.time() + except (asyncio.CancelledError, asyncio.TimeoutError): + if ( + sys.version_info >= (3, 11) + and (t := asyncio.current_task()) + and t.cancelling() + ): + raise + + # if payload still uncompleted + if not payload.is_eof() and not self._force_close: + self.log_debug("Uncompleted request.") + self.close() + + payload.set_exception(_PAYLOAD_ACCESS_ERROR) + + except asyncio.CancelledError: + self.log_debug("Ignored premature client disconnection") + self.force_close() + raise + except Exception as exc: + self.log_exception("Unhandled exception", exc_info=exc) + self.force_close() + except BaseException: + self.force_close() + raise + finally: + request._task = None # type: ignore[assignment] # Break reference cycle in case of exception + if self.transport is None and resp is not None: + self.log_debug("Ignored premature client disconnection.") + + if self._keepalive and not self._close and not self._force_close: + # start keep-alive timer + close_time = loop.time() + keepalive_timeout + self._next_keepalive_close_time = close_time + if self._keepalive_handle is None: + self._keepalive_handle = loop.call_at( + close_time, self._process_keepalive + ) + else: + break + + # remove handler, close transport if no handlers left + if not self._force_close: + self._task_handler = None + if self.transport is not None: + self.transport.close() + + async def finish_response( + self, request: BaseRequest, resp: StreamResponse, start_time: Optional[float] + ) -> Tuple[StreamResponse, bool]: + """Prepare the response and write_eof, then log access. + + This has to + be called within the context of any exception so the access logger + can get exception information. Returns True if the client disconnects + prematurely. + """ + request._finish() + if self._request_parser is not None: + self._request_parser.set_upgraded(False) + self._upgrade = False + if self._message_tail: + self._request_parser.feed_data(self._message_tail) + self._message_tail = b"" + try: + prepare_meth = resp.prepare + except AttributeError: + if resp is None: + self.log_exception("Missing return statement on request handler") + else: + self.log_exception( + "Web-handler should return a response instance, " + "got {!r}".format(resp) + ) + exc = HTTPInternalServerError() + resp = Response( + status=exc.status, reason=exc.reason, text=exc.text, headers=exc.headers + ) + prepare_meth = resp.prepare + try: + await prepare_meth(request) + await resp.write_eof() + except ConnectionError: + self.log_access(request, resp, start_time) + return resp, True + + self.log_access(request, resp, start_time) + return resp, False + + def handle_error( + self, + request: BaseRequest, + status: int = 500, + exc: Optional[BaseException] = None, + message: Optional[str] = None, + ) -> StreamResponse: + """Handle errors. + + Returns HTTP response with specific status code. Logs additional + information. It always closes current connection. + """ + if self._request_count == 1 and isinstance(exc, BadHttpMethod): + # BadHttpMethod is common when a client sends non-HTTP + # or encrypted traffic to an HTTP port. This is expected + # to happen when connected to the public internet so we log + # it at the debug level as to not fill logs with noise. + self.logger.debug( + "Error handling request from %s", request.remote, exc_info=exc + ) + else: + self.log_exception( + "Error handling request from %s", request.remote, exc_info=exc + ) + + # some data already got sent, connection is broken + if request.writer.output_size > 0: + raise ConnectionError( + "Response is sent already, cannot send another response " + "with the error message" + ) + + ct = "text/plain" + if status == HTTPStatus.INTERNAL_SERVER_ERROR: + title = "{0.value} {0.phrase}".format(HTTPStatus.INTERNAL_SERVER_ERROR) + msg = HTTPStatus.INTERNAL_SERVER_ERROR.description + tb = None + if self.debug: + with suppress(Exception): + tb = traceback.format_exc() + + if "text/html" in request.headers.get("Accept", ""): + if tb: + tb = html_escape(tb) + msg = f"

Traceback:

\n
{tb}
" + message = ( + "" + "{title}" + "\n

{title}

" + "\n{msg}\n\n" + ).format(title=title, msg=msg) + ct = "text/html" + else: + if tb: + msg = tb + message = title + "\n\n" + msg + + resp = Response(status=status, text=message, content_type=ct) + resp.force_close() + + return resp + + def _make_error_handler( + self, err_info: _ErrInfo + ) -> Callable[[BaseRequest], Awaitable[StreamResponse]]: + async def handler(request: BaseRequest) -> StreamResponse: + return self.handle_error( + request, err_info.status, err_info.exc, err_info.message + ) + + return handler diff --git a/lib/python3.1/site-packages/aiohttp/web_request.py b/lib/python3.1/site-packages/aiohttp/web_request.py new file mode 100644 index 0000000000000000000000000000000000000000..0bc69b74db94f388fec4035ff5789bbf9ff98445 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web_request.py @@ -0,0 +1,916 @@ +import asyncio +import datetime +import io +import re +import socket +import string +import tempfile +import types +import warnings +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Final, + Iterator, + Mapping, + MutableMapping, + Optional, + Pattern, + Tuple, + Union, + cast, +) +from urllib.parse import parse_qsl + +import attr +from multidict import ( + CIMultiDict, + CIMultiDictProxy, + MultiDict, + MultiDictProxy, + MultiMapping, +) +from yarl import URL + +from . import hdrs +from ._cookie_helpers import parse_cookie_header +from .abc import AbstractStreamWriter +from .helpers import ( + _SENTINEL, + DEBUG, + ETAG_ANY, + LIST_QUOTED_ETAG_RE, + ChainMapProxy, + ETag, + HeadersMixin, + parse_http_date, + reify, + sentinel, + set_exception, +) +from .http_parser import RawRequestMessage +from .http_writer import HttpVersion +from .multipart import BodyPartReader, MultipartReader +from .streams import EmptyStreamReader, StreamReader +from .typedefs import ( + DEFAULT_JSON_DECODER, + JSONDecoder, + LooseHeaders, + RawHeaders, + StrOrURL, +) +from .web_exceptions import HTTPRequestEntityTooLarge +from .web_response import StreamResponse + +__all__ = ("BaseRequest", "FileField", "Request") + + +if TYPE_CHECKING: + from .web_app import Application + from .web_protocol import RequestHandler + from .web_urldispatcher import UrlMappingMatchInfo + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class FileField: + name: str + filename: str + file: io.BufferedReader + content_type: str + headers: CIMultiDictProxy[str] + + +_TCHAR: Final[str] = string.digits + string.ascii_letters + r"!#$%&'*+.^_`|~-" +# '-' at the end to prevent interpretation as range in a char class + +_TOKEN: Final[str] = rf"[{_TCHAR}]+" + +_QDTEXT: Final[str] = r"[{}]".format( + r"".join(chr(c) for c in (0x09, 0x20, 0x21) + tuple(range(0x23, 0x7F))) +) +# qdtext includes 0x5C to escape 0x5D ('\]') +# qdtext excludes obs-text (because obsoleted, and encoding not specified) + +_QUOTED_PAIR: Final[str] = r"\\[\t !-~]" + +_QUOTED_STRING: Final[str] = r'"(?:{quoted_pair}|{qdtext})*"'.format( + qdtext=_QDTEXT, quoted_pair=_QUOTED_PAIR +) + +_FORWARDED_PAIR: Final[str] = ( + r"({token})=({token}|{quoted_string})(:\d{{1,4}})?".format( + token=_TOKEN, quoted_string=_QUOTED_STRING + ) +) + +_QUOTED_PAIR_REPLACE_RE: Final[Pattern[str]] = re.compile(r"\\([\t !-~])") +# same pattern as _QUOTED_PAIR but contains a capture group + +_FORWARDED_PAIR_RE: Final[Pattern[str]] = re.compile(_FORWARDED_PAIR) + +############################################################ +# HTTP Request +############################################################ + + +class BaseRequest(MutableMapping[str, Any], HeadersMixin): + + POST_METHODS = { + hdrs.METH_PATCH, + hdrs.METH_POST, + hdrs.METH_PUT, + hdrs.METH_TRACE, + hdrs.METH_DELETE, + } + + ATTRS = HeadersMixin.ATTRS | frozenset( + [ + "_message", + "_protocol", + "_payload_writer", + "_payload", + "_headers", + "_method", + "_version", + "_rel_url", + "_post", + "_read_bytes", + "_state", + "_cache", + "_task", + "_client_max_size", + "_loop", + "_transport_sslcontext", + "_transport_peername", + ] + ) + _post: Optional[MultiDictProxy[Union[str, bytes, FileField]]] = None + _read_bytes: Optional[bytes] = None + + def __init__( + self, + message: RawRequestMessage, + payload: StreamReader, + protocol: "RequestHandler", + payload_writer: AbstractStreamWriter, + task: "asyncio.Task[None]", + loop: asyncio.AbstractEventLoop, + *, + client_max_size: int = 1024**2, + state: Optional[Dict[str, Any]] = None, + scheme: Optional[str] = None, + host: Optional[str] = None, + remote: Optional[str] = None, + ) -> None: + self._message = message + self._protocol = protocol + self._payload_writer = payload_writer + + self._payload = payload + self._headers: CIMultiDictProxy[str] = message.headers + self._method = message.method + self._version = message.version + self._cache: Dict[str, Any] = {} + url = message.url + if url.absolute: + if scheme is not None: + url = url.with_scheme(scheme) + if host is not None: + url = url.with_host(host) + # absolute URL is given, + # override auto-calculating url, host, and scheme + # all other properties should be good + self._cache["url"] = url + self._cache["host"] = url.host + self._cache["scheme"] = url.scheme + self._rel_url = url.relative() + else: + self._rel_url = url + if scheme is not None: + self._cache["scheme"] = scheme + if host is not None: + self._cache["host"] = host + + self._state = {} if state is None else state + self._task = task + self._client_max_size = client_max_size + self._loop = loop + + self._transport_sslcontext = protocol.ssl_context + self._transport_peername = protocol.peername + + if remote is not None: + self._cache["remote"] = remote + + def clone( + self, + *, + method: Union[str, _SENTINEL] = sentinel, + rel_url: Union[StrOrURL, _SENTINEL] = sentinel, + headers: Union[LooseHeaders, _SENTINEL] = sentinel, + scheme: Union[str, _SENTINEL] = sentinel, + host: Union[str, _SENTINEL] = sentinel, + remote: Union[str, _SENTINEL] = sentinel, + client_max_size: Union[int, _SENTINEL] = sentinel, + ) -> "BaseRequest": + """Clone itself with replacement some attributes. + + Creates and returns a new instance of Request object. If no parameters + are given, an exact copy is returned. If a parameter is not passed, it + will reuse the one from the current request object. + """ + if self._read_bytes: + raise RuntimeError("Cannot clone request after reading its content") + + dct: Dict[str, Any] = {} + if method is not sentinel: + dct["method"] = method + if rel_url is not sentinel: + new_url: URL = URL(rel_url) + dct["url"] = new_url + dct["path"] = str(new_url) + if headers is not sentinel: + # a copy semantic + dct["headers"] = CIMultiDictProxy(CIMultiDict(headers)) + dct["raw_headers"] = tuple( + (k.encode("utf-8"), v.encode("utf-8")) + for k, v in dct["headers"].items() + ) + + message = self._message._replace(**dct) + + kwargs = {} + if scheme is not sentinel: + kwargs["scheme"] = scheme + if host is not sentinel: + kwargs["host"] = host + if remote is not sentinel: + kwargs["remote"] = remote + if client_max_size is sentinel: + client_max_size = self._client_max_size + + return self.__class__( + message, + self._payload, + self._protocol, + self._payload_writer, + self._task, + self._loop, + client_max_size=client_max_size, + state=self._state.copy(), + **kwargs, + ) + + @property + def task(self) -> "asyncio.Task[None]": + return self._task + + @property + def protocol(self) -> "RequestHandler": + return self._protocol + + @property + def transport(self) -> Optional[asyncio.Transport]: + if self._protocol is None: + return None + return self._protocol.transport + + @property + def writer(self) -> AbstractStreamWriter: + return self._payload_writer + + @property + def client_max_size(self) -> int: + return self._client_max_size + + @reify + def message(self) -> RawRequestMessage: + warnings.warn("Request.message is deprecated", DeprecationWarning, stacklevel=3) + return self._message + + @reify + def rel_url(self) -> URL: + return self._rel_url + + @reify + def loop(self) -> asyncio.AbstractEventLoop: + warnings.warn( + "request.loop property is deprecated", DeprecationWarning, stacklevel=2 + ) + return self._loop + + # MutableMapping API + + def __getitem__(self, key: str) -> Any: + return self._state[key] + + def __setitem__(self, key: str, value: Any) -> None: + self._state[key] = value + + def __delitem__(self, key: str) -> None: + del self._state[key] + + def __len__(self) -> int: + return len(self._state) + + def __iter__(self) -> Iterator[str]: + return iter(self._state) + + ######## + + @reify + def secure(self) -> bool: + """A bool indicating if the request is handled with SSL.""" + return self.scheme == "https" + + @reify + def forwarded(self) -> Tuple[Mapping[str, str], ...]: + """A tuple containing all parsed Forwarded header(s). + + Makes an effort to parse Forwarded headers as specified by RFC 7239: + + - It adds one (immutable) dictionary per Forwarded 'field-value', ie + per proxy. The element corresponds to the data in the Forwarded + field-value added by the first proxy encountered by the client. Each + subsequent item corresponds to those added by later proxies. + - It checks that every value has valid syntax in general as specified + in section 4: either a 'token' or a 'quoted-string'. + - It un-escapes found escape sequences. + - It does NOT validate 'by' and 'for' contents as specified in section + 6. + - It does NOT validate 'host' contents (Host ABNF). + - It does NOT validate 'proto' contents for valid URI scheme names. + + Returns a tuple containing one or more immutable dicts + """ + elems = [] + for field_value in self._message.headers.getall(hdrs.FORWARDED, ()): + length = len(field_value) + pos = 0 + need_separator = False + elem: Dict[str, str] = {} + elems.append(types.MappingProxyType(elem)) + while 0 <= pos < length: + match = _FORWARDED_PAIR_RE.match(field_value, pos) + if match is not None: # got a valid forwarded-pair + if need_separator: + # bad syntax here, skip to next comma + pos = field_value.find(",", pos) + else: + name, value, port = match.groups() + if value[0] == '"': + # quoted string: remove quotes and unescape + value = _QUOTED_PAIR_REPLACE_RE.sub(r"\1", value[1:-1]) + if port: + value += port + elem[name.lower()] = value + pos += len(match.group(0)) + need_separator = True + elif field_value[pos] == ",": # next forwarded-element + need_separator = False + elem = {} + elems.append(types.MappingProxyType(elem)) + pos += 1 + elif field_value[pos] == ";": # next forwarded-pair + need_separator = False + pos += 1 + elif field_value[pos] in " \t": + # Allow whitespace even between forwarded-pairs, though + # RFC 7239 doesn't. This simplifies code and is in line + # with Postel's law. + pos += 1 + else: + # bad syntax here, skip to next comma + pos = field_value.find(",", pos) + return tuple(elems) + + @reify + def scheme(self) -> str: + """A string representing the scheme of the request. + + Hostname is resolved in this order: + + - overridden value by .clone(scheme=new_scheme) call. + - type of connection to peer: HTTPS if socket is SSL, HTTP otherwise. + + 'http' or 'https'. + """ + if self._transport_sslcontext: + return "https" + else: + return "http" + + @reify + def method(self) -> str: + """Read only property for getting HTTP method. + + The value is upper-cased str like 'GET', 'POST', 'PUT' etc. + """ + return self._method + + @reify + def version(self) -> HttpVersion: + """Read only property for getting HTTP version of request. + + Returns aiohttp.protocol.HttpVersion instance. + """ + return self._version + + @reify + def host(self) -> str: + """Hostname of the request. + + Hostname is resolved in this order: + + - overridden value by .clone(host=new_host) call. + - HOST HTTP header + - socket.getfqdn() value + + For example, 'example.com' or 'localhost:8080'. + + For historical reasons, the port number may be included. + """ + host = self._message.headers.get(hdrs.HOST) + if host is not None: + return host + return socket.getfqdn() + + @reify + def remote(self) -> Optional[str]: + """Remote IP of client initiated HTTP request. + + The IP is resolved in this order: + + - overridden value by .clone(remote=new_remote) call. + - peername of opened socket + """ + if self._transport_peername is None: + return None + if isinstance(self._transport_peername, (list, tuple)): + return str(self._transport_peername[0]) + return str(self._transport_peername) + + @reify + def url(self) -> URL: + """The full URL of the request.""" + # authority is used here because it may include the port number + # and we want yarl to parse it correctly + return URL.build(scheme=self.scheme, authority=self.host).join(self._rel_url) + + @reify + def path(self) -> str: + """The URL including *PATH INFO* without the host or scheme. + + E.g., ``/app/blog`` + """ + return self._rel_url.path + + @reify + def path_qs(self) -> str: + """The URL including PATH_INFO and the query string. + + E.g, /app/blog?id=10 + """ + return str(self._rel_url) + + @reify + def raw_path(self) -> str: + """The URL including raw *PATH INFO* without the host or scheme. + + Warning, the path is unquoted and may contains non valid URL characters + + E.g., ``/my%2Fpath%7Cwith%21some%25strange%24characters`` + """ + return self._message.path + + @reify + def query(self) -> "MultiMapping[str]": + """A multidict with all the variables in the query string.""" + return self._rel_url.query + + @reify + def query_string(self) -> str: + """The query string in the URL. + + E.g., id=10 + """ + return self._rel_url.query_string + + @reify + def headers(self) -> CIMultiDictProxy[str]: + """A case-insensitive multidict proxy with all headers.""" + return self._headers + + @reify + def raw_headers(self) -> RawHeaders: + """A sequence of pairs for all headers.""" + return self._message.raw_headers + + @reify + def if_modified_since(self) -> Optional[datetime.datetime]: + """The value of If-Modified-Since HTTP header, or None. + + This header is represented as a `datetime` object. + """ + return parse_http_date(self.headers.get(hdrs.IF_MODIFIED_SINCE)) + + @reify + def if_unmodified_since(self) -> Optional[datetime.datetime]: + """The value of If-Unmodified-Since HTTP header, or None. + + This header is represented as a `datetime` object. + """ + return parse_http_date(self.headers.get(hdrs.IF_UNMODIFIED_SINCE)) + + @staticmethod + def _etag_values(etag_header: str) -> Iterator[ETag]: + """Extract `ETag` objects from raw header.""" + if etag_header == ETAG_ANY: + yield ETag( + is_weak=False, + value=ETAG_ANY, + ) + else: + for match in LIST_QUOTED_ETAG_RE.finditer(etag_header): + is_weak, value, garbage = match.group(2, 3, 4) + # Any symbol captured by 4th group means + # that the following sequence is invalid. + if garbage: + break + + yield ETag( + is_weak=bool(is_weak), + value=value, + ) + + @classmethod + def _if_match_or_none_impl( + cls, header_value: Optional[str] + ) -> Optional[Tuple[ETag, ...]]: + if not header_value: + return None + + return tuple(cls._etag_values(header_value)) + + @reify + def if_match(self) -> Optional[Tuple[ETag, ...]]: + """The value of If-Match HTTP header, or None. + + This header is represented as a `tuple` of `ETag` objects. + """ + return self._if_match_or_none_impl(self.headers.get(hdrs.IF_MATCH)) + + @reify + def if_none_match(self) -> Optional[Tuple[ETag, ...]]: + """The value of If-None-Match HTTP header, or None. + + This header is represented as a `tuple` of `ETag` objects. + """ + return self._if_match_or_none_impl(self.headers.get(hdrs.IF_NONE_MATCH)) + + @reify + def if_range(self) -> Optional[datetime.datetime]: + """The value of If-Range HTTP header, or None. + + This header is represented as a `datetime` object. + """ + return parse_http_date(self.headers.get(hdrs.IF_RANGE)) + + @reify + def keep_alive(self) -> bool: + """Is keepalive enabled by client?""" + return not self._message.should_close + + @reify + def cookies(self) -> Mapping[str, str]: + """Return request cookies. + + A read-only dictionary-like object. + """ + # Use parse_cookie_header for RFC 6265 compliant Cookie header parsing + # that accepts special characters in cookie names (fixes #2683) + parsed = parse_cookie_header(self.headers.get(hdrs.COOKIE, "")) + # Extract values from Morsel objects + return MappingProxyType({name: morsel.value for name, morsel in parsed}) + + @reify + def http_range(self) -> slice: + """The content of Range HTTP header. + + Return a slice instance. + + """ + rng = self._headers.get(hdrs.RANGE) + start, end = None, None + if rng is not None: + try: + pattern = r"^bytes=(\d*)-(\d*)$" + start, end = re.findall(pattern, rng)[0] + except IndexError: # pattern was not found in header + raise ValueError("range not in acceptable format") + + end = int(end) if end else None + start = int(start) if start else None + + if start is None and end is not None: + # end with no start is to return tail of content + start = -end + end = None + + if start is not None and end is not None: + # end is inclusive in range header, exclusive for slice + end += 1 + + if start >= end: + raise ValueError("start cannot be after end") + + if start is end is None: # No valid range supplied + raise ValueError("No start or end of range specified") + + return slice(start, end, 1) + + @reify + def content(self) -> StreamReader: + """Return raw payload stream.""" + return self._payload + + @property + def has_body(self) -> bool: + """Return True if request's HTTP BODY can be read, False otherwise.""" + warnings.warn( + "Deprecated, use .can_read_body #2005", DeprecationWarning, stacklevel=2 + ) + return not self._payload.at_eof() + + @property + def can_read_body(self) -> bool: + """Return True if request's HTTP BODY can be read, False otherwise.""" + return not self._payload.at_eof() + + @reify + def body_exists(self) -> bool: + """Return True if request has HTTP BODY, False otherwise.""" + return type(self._payload) is not EmptyStreamReader + + async def release(self) -> None: + """Release request. + + Eat unread part of HTTP BODY if present. + """ + while not self._payload.at_eof(): + await self._payload.readany() + + async def read(self) -> bytes: + """Read request body if present. + + Returns bytes object with full request content. + """ + if self._read_bytes is None: + body = bytearray() + while True: + chunk = await self._payload.readany() + body.extend(chunk) + if self._client_max_size: + body_size = len(body) + if body_size >= self._client_max_size: + raise HTTPRequestEntityTooLarge( + max_size=self._client_max_size, actual_size=body_size + ) + if not chunk: + break + self._read_bytes = bytes(body) + return self._read_bytes + + async def text(self) -> str: + """Return BODY as text using encoding from .charset.""" + bytes_body = await self.read() + encoding = self.charset or "utf-8" + return bytes_body.decode(encoding) + + async def json(self, *, loads: JSONDecoder = DEFAULT_JSON_DECODER) -> Any: + """Return BODY as JSON.""" + body = await self.text() + return loads(body) + + async def multipart(self) -> MultipartReader: + """Return async iterator to process BODY as multipart.""" + return MultipartReader(self._headers, self._payload) + + async def post(self) -> "MultiDictProxy[Union[str, bytes, FileField]]": + """Return POST parameters.""" + if self._post is not None: + return self._post + if self._method not in self.POST_METHODS: + self._post = MultiDictProxy(MultiDict()) + return self._post + + content_type = self.content_type + if content_type not in ( + "", + "application/x-www-form-urlencoded", + "multipart/form-data", + ): + self._post = MultiDictProxy(MultiDict()) + return self._post + + out: MultiDict[Union[str, bytes, FileField]] = MultiDict() + + if content_type == "multipart/form-data": + multipart = await self.multipart() + max_size = self._client_max_size + + field = await multipart.next() + while field is not None: + size = 0 + field_ct = field.headers.get(hdrs.CONTENT_TYPE) + + if isinstance(field, BodyPartReader): + assert field.name is not None + + # Note that according to RFC 7578, the Content-Type header + # is optional, even for files, so we can't assume it's + # present. + # https://tools.ietf.org/html/rfc7578#section-4.4 + if field.filename: + # store file in temp file + tmp = await self._loop.run_in_executor( + None, tempfile.TemporaryFile + ) + chunk = await field.read_chunk(size=2**16) + while chunk: + chunk = field.decode(chunk) + await self._loop.run_in_executor(None, tmp.write, chunk) + size += len(chunk) + if 0 < max_size < size: + await self._loop.run_in_executor(None, tmp.close) + raise HTTPRequestEntityTooLarge( + max_size=max_size, actual_size=size + ) + chunk = await field.read_chunk(size=2**16) + await self._loop.run_in_executor(None, tmp.seek, 0) + + if field_ct is None: + field_ct = "application/octet-stream" + + ff = FileField( + field.name, + field.filename, + cast(io.BufferedReader, tmp), + field_ct, + field.headers, + ) + out.add(field.name, ff) + else: + # deal with ordinary data + value = await field.read(decode=True) + if field_ct is None or field_ct.startswith("text/"): + charset = field.get_charset(default="utf-8") + out.add(field.name, value.decode(charset)) + else: + out.add(field.name, value) + size += len(value) + if 0 < max_size < size: + raise HTTPRequestEntityTooLarge( + max_size=max_size, actual_size=size + ) + else: + raise ValueError( + "To decode nested multipart you need to use custom reader", + ) + + field = await multipart.next() + else: + data = await self.read() + if data: + charset = self.charset or "utf-8" + out.extend( + parse_qsl( + data.rstrip().decode(charset), + keep_blank_values=True, + encoding=charset, + ) + ) + + self._post = MultiDictProxy(out) + return self._post + + def get_extra_info(self, name: str, default: Any = None) -> Any: + """Extra info from protocol transport""" + protocol = self._protocol + if protocol is None: + return default + + transport = protocol.transport + if transport is None: + return default + + return transport.get_extra_info(name, default) + + def __repr__(self) -> str: + ascii_encodable_path = self.path.encode("ascii", "backslashreplace").decode( + "ascii" + ) + return "<{} {} {} >".format( + self.__class__.__name__, self._method, ascii_encodable_path + ) + + def __eq__(self, other: object) -> bool: + return id(self) == id(other) + + def __bool__(self) -> bool: + return True + + async def _prepare_hook(self, response: StreamResponse) -> None: + return + + def _cancel(self, exc: BaseException) -> None: + set_exception(self._payload, exc) + + def _finish(self) -> None: + if self._post is None or self.content_type != "multipart/form-data": + return + + # NOTE: Release file descriptors for the + # NOTE: `tempfile.Temporaryfile`-created `_io.BufferedRandom` + # NOTE: instances of files sent within multipart request body + # NOTE: via HTTP POST request. + for file_name, file_field_object in self._post.items(): + if isinstance(file_field_object, FileField): + file_field_object.file.close() + + +class Request(BaseRequest): + + ATTRS = BaseRequest.ATTRS | frozenset(["_match_info"]) + + _match_info: Optional["UrlMappingMatchInfo"] = None + + if DEBUG: + + def __setattr__(self, name: str, val: Any) -> None: + if name not in self.ATTRS: + warnings.warn( + "Setting custom {}.{} attribute " + "is discouraged".format(self.__class__.__name__, name), + DeprecationWarning, + stacklevel=2, + ) + super().__setattr__(name, val) + + def clone( + self, + *, + method: Union[str, _SENTINEL] = sentinel, + rel_url: Union[StrOrURL, _SENTINEL] = sentinel, + headers: Union[LooseHeaders, _SENTINEL] = sentinel, + scheme: Union[str, _SENTINEL] = sentinel, + host: Union[str, _SENTINEL] = sentinel, + remote: Union[str, _SENTINEL] = sentinel, + client_max_size: Union[int, _SENTINEL] = sentinel, + ) -> "Request": + ret = super().clone( + method=method, + rel_url=rel_url, + headers=headers, + scheme=scheme, + host=host, + remote=remote, + client_max_size=client_max_size, + ) + new_ret = cast(Request, ret) + new_ret._match_info = self._match_info + return new_ret + + @reify + def match_info(self) -> "UrlMappingMatchInfo": + """Result of route resolving.""" + match_info = self._match_info + assert match_info is not None + return match_info + + @property + def app(self) -> "Application": + """Application instance.""" + match_info = self._match_info + assert match_info is not None + return match_info.current_app + + @property + def config_dict(self) -> ChainMapProxy: + match_info = self._match_info + assert match_info is not None + lst = match_info.apps + app = self.app + idx = lst.index(app) + sublist = list(reversed(lst[: idx + 1])) + return ChainMapProxy(sublist) + + async def _prepare_hook(self, response: StreamResponse) -> None: + match_info = self._match_info + if match_info is None: + return + for app in match_info._apps: + if on_response_prepare := app.on_response_prepare: + await on_response_prepare.send(self, response) diff --git a/lib/python3.1/site-packages/aiohttp/web_response.py b/lib/python3.1/site-packages/aiohttp/web_response.py new file mode 100644 index 0000000000000000000000000000000000000000..e5f8b6cd652c36274386cb0ce5586d8a5fdf044c --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web_response.py @@ -0,0 +1,856 @@ +import asyncio +import collections.abc +import datetime +import enum +import json +import math +import time +import warnings +from concurrent.futures import Executor +from http import HTTPStatus +from http.cookies import SimpleCookie +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterator, + MutableMapping, + Optional, + Union, + cast, +) + +from multidict import CIMultiDict, istr + +from . import hdrs, payload +from .abc import AbstractStreamWriter +from .compression_utils import ZLibCompressor +from .helpers import ( + ETAG_ANY, + QUOTED_ETAG_RE, + ETag, + HeadersMixin, + must_be_empty_body, + parse_http_date, + rfc822_formatted_time, + sentinel, + should_remove_content_length, + validate_etag_value, +) +from .http import SERVER_SOFTWARE, HttpVersion10, HttpVersion11 +from .payload import Payload +from .typedefs import JSONEncoder, LooseHeaders + +REASON_PHRASES = {http_status.value: http_status.phrase for http_status in HTTPStatus} +LARGE_BODY_SIZE = 1024**2 + +__all__ = ("ContentCoding", "StreamResponse", "Response", "json_response") + + +if TYPE_CHECKING: + from .web_request import BaseRequest + + BaseClass = MutableMapping[str, Any] +else: + BaseClass = collections.abc.MutableMapping + + +# TODO(py311): Convert to StrEnum for wider use +class ContentCoding(enum.Enum): + # The content codings that we have support for. + # + # Additional registered codings are listed at: + # https://www.iana.org/assignments/http-parameters/http-parameters.xhtml#content-coding + deflate = "deflate" + gzip = "gzip" + identity = "identity" + + +CONTENT_CODINGS = {coding.value: coding for coding in ContentCoding} + +############################################################ +# HTTP Response classes +############################################################ + + +class StreamResponse(BaseClass, HeadersMixin): + + _body: Union[None, bytes, bytearray, Payload] + _length_check = True + _body = None + _keep_alive: Optional[bool] = None + _chunked: bool = False + _compression: bool = False + _compression_strategy: Optional[int] = None + _compression_force: Optional[ContentCoding] = None + _req: Optional["BaseRequest"] = None + _payload_writer: Optional[AbstractStreamWriter] = None + _eof_sent: bool = False + _must_be_empty_body: Optional[bool] = None + _body_length = 0 + _cookies: Optional[SimpleCookie] = None + _send_headers_immediately = True + + def __init__( + self, + *, + status: int = 200, + reason: Optional[str] = None, + headers: Optional[LooseHeaders] = None, + _real_headers: Optional[CIMultiDict[str]] = None, + ) -> None: + """Initialize a new stream response object. + + _real_headers is an internal parameter used to pass a pre-populated + headers object. It is used by the `Response` class to avoid copying + the headers when creating a new response object. It is not intended + to be used by external code. + """ + self._state: Dict[str, Any] = {} + + if _real_headers is not None: + self._headers = _real_headers + elif headers is not None: + self._headers: CIMultiDict[str] = CIMultiDict(headers) + else: + self._headers = CIMultiDict() + + self._set_status(status, reason) + + @property + def prepared(self) -> bool: + return self._eof_sent or self._payload_writer is not None + + @property + def task(self) -> "Optional[asyncio.Task[None]]": + if self._req: + return self._req.task + else: + return None + + @property + def status(self) -> int: + return self._status + + @property + def chunked(self) -> bool: + return self._chunked + + @property + def compression(self) -> bool: + return self._compression + + @property + def reason(self) -> str: + return self._reason + + def set_status( + self, + status: int, + reason: Optional[str] = None, + ) -> None: + assert ( + not self.prepared + ), "Cannot change the response status code after the headers have been sent" + self._set_status(status, reason) + + def _set_status(self, status: int, reason: Optional[str]) -> None: + self._status = int(status) + if reason is None: + reason = REASON_PHRASES.get(self._status, "") + elif "\n" in reason: + raise ValueError("Reason cannot contain \\n") + self._reason = reason + + @property + def keep_alive(self) -> Optional[bool]: + return self._keep_alive + + def force_close(self) -> None: + self._keep_alive = False + + @property + def body_length(self) -> int: + return self._body_length + + @property + def output_length(self) -> int: + warnings.warn("output_length is deprecated", DeprecationWarning) + assert self._payload_writer + return self._payload_writer.buffer_size + + def enable_chunked_encoding(self, chunk_size: Optional[int] = None) -> None: + """Enables automatic chunked transfer encoding.""" + if hdrs.CONTENT_LENGTH in self._headers: + raise RuntimeError( + "You can't enable chunked encoding when a content length is set" + ) + if chunk_size is not None: + warnings.warn("Chunk size is deprecated #1615", DeprecationWarning) + self._chunked = True + + def enable_compression( + self, + force: Optional[Union[bool, ContentCoding]] = None, + strategy: Optional[int] = None, + ) -> None: + """Enables response compression encoding.""" + # Backwards compatibility for when force was a bool <0.17. + if isinstance(force, bool): + force = ContentCoding.deflate if force else ContentCoding.identity + warnings.warn( + "Using boolean for force is deprecated #3318", DeprecationWarning + ) + elif force is not None: + assert isinstance( + force, ContentCoding + ), "force should one of None, bool or ContentEncoding" + + self._compression = True + self._compression_force = force + self._compression_strategy = strategy + + @property + def headers(self) -> "CIMultiDict[str]": + return self._headers + + @property + def cookies(self) -> SimpleCookie: + if self._cookies is None: + self._cookies = SimpleCookie() + return self._cookies + + def set_cookie( + self, + name: str, + value: str, + *, + expires: Optional[str] = None, + domain: Optional[str] = None, + max_age: Optional[Union[int, str]] = None, + path: str = "/", + secure: Optional[bool] = None, + httponly: Optional[bool] = None, + version: Optional[str] = None, + samesite: Optional[str] = None, + partitioned: Optional[bool] = None, + ) -> None: + """Set or update response cookie. + + Sets new cookie or updates existent with new value. + Also updates only those params which are not None. + """ + if self._cookies is None: + self._cookies = SimpleCookie() + + self._cookies[name] = value + c = self._cookies[name] + + if expires is not None: + c["expires"] = expires + elif c.get("expires") == "Thu, 01 Jan 1970 00:00:00 GMT": + del c["expires"] + + if domain is not None: + c["domain"] = domain + + if max_age is not None: + c["max-age"] = str(max_age) + elif "max-age" in c: + del c["max-age"] + + c["path"] = path + + if secure is not None: + c["secure"] = secure + if httponly is not None: + c["httponly"] = httponly + if version is not None: + c["version"] = version + if samesite is not None: + c["samesite"] = samesite + + if partitioned is not None: + c["partitioned"] = partitioned + + def del_cookie( + self, + name: str, + *, + domain: Optional[str] = None, + path: str = "/", + secure: Optional[bool] = None, + httponly: Optional[bool] = None, + samesite: Optional[str] = None, + ) -> None: + """Delete cookie. + + Creates new empty expired cookie. + """ + # TODO: do we need domain/path here? + if self._cookies is not None: + self._cookies.pop(name, None) + self.set_cookie( + name, + "", + max_age=0, + expires="Thu, 01 Jan 1970 00:00:00 GMT", + domain=domain, + path=path, + secure=secure, + httponly=httponly, + samesite=samesite, + ) + + @property + def content_length(self) -> Optional[int]: + # Just a placeholder for adding setter + return super().content_length + + @content_length.setter + def content_length(self, value: Optional[int]) -> None: + if value is not None: + value = int(value) + if self._chunked: + raise RuntimeError( + "You can't set content length when chunked encoding is enable" + ) + self._headers[hdrs.CONTENT_LENGTH] = str(value) + else: + self._headers.pop(hdrs.CONTENT_LENGTH, None) + + @property + def content_type(self) -> str: + # Just a placeholder for adding setter + return super().content_type + + @content_type.setter + def content_type(self, value: str) -> None: + self.content_type # read header values if needed + self._content_type = str(value) + self._generate_content_type_header() + + @property + def charset(self) -> Optional[str]: + # Just a placeholder for adding setter + return super().charset + + @charset.setter + def charset(self, value: Optional[str]) -> None: + ctype = self.content_type # read header values if needed + if ctype == "application/octet-stream": + raise RuntimeError( + "Setting charset for application/octet-stream " + "doesn't make sense, setup content_type first" + ) + assert self._content_dict is not None + if value is None: + self._content_dict.pop("charset", None) + else: + self._content_dict["charset"] = str(value).lower() + self._generate_content_type_header() + + @property + def last_modified(self) -> Optional[datetime.datetime]: + """The value of Last-Modified HTTP header, or None. + + This header is represented as a `datetime` object. + """ + return parse_http_date(self._headers.get(hdrs.LAST_MODIFIED)) + + @last_modified.setter + def last_modified( + self, value: Optional[Union[int, float, datetime.datetime, str]] + ) -> None: + if value is None: + self._headers.pop(hdrs.LAST_MODIFIED, None) + elif isinstance(value, (int, float)): + self._headers[hdrs.LAST_MODIFIED] = time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(math.ceil(value)) + ) + elif isinstance(value, datetime.datetime): + self._headers[hdrs.LAST_MODIFIED] = time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", value.utctimetuple() + ) + elif isinstance(value, str): + self._headers[hdrs.LAST_MODIFIED] = value + else: + msg = f"Unsupported type for last_modified: {type(value).__name__}" + raise TypeError(msg) + + @property + def etag(self) -> Optional[ETag]: + quoted_value = self._headers.get(hdrs.ETAG) + if not quoted_value: + return None + elif quoted_value == ETAG_ANY: + return ETag(value=ETAG_ANY) + match = QUOTED_ETAG_RE.fullmatch(quoted_value) + if not match: + return None + is_weak, value = match.group(1, 2) + return ETag( + is_weak=bool(is_weak), + value=value, + ) + + @etag.setter + def etag(self, value: Optional[Union[ETag, str]]) -> None: + if value is None: + self._headers.pop(hdrs.ETAG, None) + elif (isinstance(value, str) and value == ETAG_ANY) or ( + isinstance(value, ETag) and value.value == ETAG_ANY + ): + self._headers[hdrs.ETAG] = ETAG_ANY + elif isinstance(value, str): + validate_etag_value(value) + self._headers[hdrs.ETAG] = f'"{value}"' + elif isinstance(value, ETag) and isinstance(value.value, str): + validate_etag_value(value.value) + hdr_value = f'W/"{value.value}"' if value.is_weak else f'"{value.value}"' + self._headers[hdrs.ETAG] = hdr_value + else: + raise ValueError( + f"Unsupported etag type: {type(value)}. " + f"etag must be str, ETag or None" + ) + + def _generate_content_type_header( + self, CONTENT_TYPE: istr = hdrs.CONTENT_TYPE + ) -> None: + assert self._content_dict is not None + assert self._content_type is not None + params = "; ".join(f"{k}={v}" for k, v in self._content_dict.items()) + if params: + ctype = self._content_type + "; " + params + else: + ctype = self._content_type + self._headers[CONTENT_TYPE] = ctype + + async def _do_start_compression(self, coding: ContentCoding) -> None: + if coding is ContentCoding.identity: + return + assert self._payload_writer is not None + self._headers[hdrs.CONTENT_ENCODING] = coding.value + self._payload_writer.enable_compression( + coding.value, self._compression_strategy + ) + # Compressed payload may have different content length, + # remove the header + self._headers.popall(hdrs.CONTENT_LENGTH, None) + + async def _start_compression(self, request: "BaseRequest") -> None: + if self._compression_force: + await self._do_start_compression(self._compression_force) + return + # Encoding comparisons should be case-insensitive + # https://www.rfc-editor.org/rfc/rfc9110#section-8.4.1 + accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower() + for value, coding in CONTENT_CODINGS.items(): + if value in accept_encoding: + await self._do_start_compression(coding) + return + + async def prepare(self, request: "BaseRequest") -> Optional[AbstractStreamWriter]: + if self._eof_sent: + return None + if self._payload_writer is not None: + return self._payload_writer + self._must_be_empty_body = must_be_empty_body(request.method, self.status) + return await self._start(request) + + async def _start(self, request: "BaseRequest") -> AbstractStreamWriter: + self._req = request + writer = self._payload_writer = request._payload_writer + + await self._prepare_headers() + await request._prepare_hook(self) + await self._write_headers() + + return writer + + async def _prepare_headers(self) -> None: + request = self._req + assert request is not None + writer = self._payload_writer + assert writer is not None + keep_alive = self._keep_alive + if keep_alive is None: + keep_alive = request.keep_alive + self._keep_alive = keep_alive + + version = request.version + + headers = self._headers + if self._cookies: + for cookie in self._cookies.values(): + value = cookie.output(header="")[1:] + headers.add(hdrs.SET_COOKIE, value) + + if self._compression: + await self._start_compression(request) + + if self._chunked: + if version != HttpVersion11: + raise RuntimeError( + "Using chunked encoding is forbidden " + "for HTTP/{0.major}.{0.minor}".format(request.version) + ) + if not self._must_be_empty_body: + writer.enable_chunking() + headers[hdrs.TRANSFER_ENCODING] = "chunked" + elif self._length_check: # Disabled for WebSockets + writer.length = self.content_length + if writer.length is None: + if version >= HttpVersion11: + if not self._must_be_empty_body: + writer.enable_chunking() + headers[hdrs.TRANSFER_ENCODING] = "chunked" + elif not self._must_be_empty_body: + keep_alive = False + + # HTTP 1.1: https://tools.ietf.org/html/rfc7230#section-3.3.2 + # HTTP 1.0: https://tools.ietf.org/html/rfc1945#section-10.4 + if self._must_be_empty_body: + if hdrs.CONTENT_LENGTH in headers and should_remove_content_length( + request.method, self.status + ): + del headers[hdrs.CONTENT_LENGTH] + # https://datatracker.ietf.org/doc/html/rfc9112#section-6.1-10 + # https://datatracker.ietf.org/doc/html/rfc9112#section-6.1-13 + if hdrs.TRANSFER_ENCODING in headers: + del headers[hdrs.TRANSFER_ENCODING] + elif (writer.length if self._length_check else self.content_length) != 0: + # https://www.rfc-editor.org/rfc/rfc9110#section-8.3-5 + headers.setdefault(hdrs.CONTENT_TYPE, "application/octet-stream") + headers.setdefault(hdrs.DATE, rfc822_formatted_time()) + headers.setdefault(hdrs.SERVER, SERVER_SOFTWARE) + + # connection header + if hdrs.CONNECTION not in headers: + if keep_alive: + if version == HttpVersion10: + headers[hdrs.CONNECTION] = "keep-alive" + elif version == HttpVersion11: + headers[hdrs.CONNECTION] = "close" + + async def _write_headers(self) -> None: + request = self._req + assert request is not None + writer = self._payload_writer + assert writer is not None + # status line + version = request.version + status_line = f"HTTP/{version[0]}.{version[1]} {self._status} {self._reason}" + await writer.write_headers(status_line, self._headers) + # Send headers immediately if not opted into buffering + if self._send_headers_immediately: + writer.send_headers() + + async def write(self, data: Union[bytes, bytearray, memoryview]) -> None: + assert isinstance( + data, (bytes, bytearray, memoryview) + ), "data argument must be byte-ish (%r)" % type(data) + + if self._eof_sent: + raise RuntimeError("Cannot call write() after write_eof()") + if self._payload_writer is None: + raise RuntimeError("Cannot call write() before prepare()") + + await self._payload_writer.write(data) + + async def drain(self) -> None: + assert not self._eof_sent, "EOF has already been sent" + assert self._payload_writer is not None, "Response has not been started" + warnings.warn( + "drain method is deprecated, use await resp.write()", + DeprecationWarning, + stacklevel=2, + ) + await self._payload_writer.drain() + + async def write_eof(self, data: bytes = b"") -> None: + assert isinstance( + data, (bytes, bytearray, memoryview) + ), "data argument must be byte-ish (%r)" % type(data) + + if self._eof_sent: + return + + assert self._payload_writer is not None, "Response has not been started" + + await self._payload_writer.write_eof(data) + self._eof_sent = True + self._req = None + self._body_length = self._payload_writer.output_size + self._payload_writer = None + + def __repr__(self) -> str: + if self._eof_sent: + info = "eof" + elif self.prepared: + assert self._req is not None + info = f"{self._req.method} {self._req.path} " + else: + info = "not prepared" + return f"<{self.__class__.__name__} {self.reason} {info}>" + + def __getitem__(self, key: str) -> Any: + return self._state[key] + + def __setitem__(self, key: str, value: Any) -> None: + self._state[key] = value + + def __delitem__(self, key: str) -> None: + del self._state[key] + + def __len__(self) -> int: + return len(self._state) + + def __iter__(self) -> Iterator[str]: + return iter(self._state) + + def __hash__(self) -> int: + return hash(id(self)) + + def __eq__(self, other: object) -> bool: + return self is other + + def __bool__(self) -> bool: + return True + + +class Response(StreamResponse): + + _compressed_body: Optional[bytes] = None + _send_headers_immediately = False + + def __init__( + self, + *, + body: Any = None, + status: int = 200, + reason: Optional[str] = None, + text: Optional[str] = None, + headers: Optional[LooseHeaders] = None, + content_type: Optional[str] = None, + charset: Optional[str] = None, + zlib_executor_size: Optional[int] = None, + zlib_executor: Optional[Executor] = None, + ) -> None: + if body is not None and text is not None: + raise ValueError("body and text are not allowed together") + + if headers is None: + real_headers: CIMultiDict[str] = CIMultiDict() + else: + real_headers = CIMultiDict(headers) + + if content_type is not None and "charset" in content_type: + raise ValueError("charset must not be in content_type argument") + + if text is not None: + if hdrs.CONTENT_TYPE in real_headers: + if content_type or charset: + raise ValueError( + "passing both Content-Type header and " + "content_type or charset params " + "is forbidden" + ) + else: + # fast path for filling headers + if not isinstance(text, str): + raise TypeError("text argument must be str (%r)" % type(text)) + if content_type is None: + content_type = "text/plain" + if charset is None: + charset = "utf-8" + real_headers[hdrs.CONTENT_TYPE] = content_type + "; charset=" + charset + body = text.encode(charset) + text = None + elif hdrs.CONTENT_TYPE in real_headers: + if content_type is not None or charset is not None: + raise ValueError( + "passing both Content-Type header and " + "content_type or charset params " + "is forbidden" + ) + elif content_type is not None: + if charset is not None: + content_type += "; charset=" + charset + real_headers[hdrs.CONTENT_TYPE] = content_type + + super().__init__(status=status, reason=reason, _real_headers=real_headers) + + if text is not None: + self.text = text + else: + self.body = body + + self._zlib_executor_size = zlib_executor_size + self._zlib_executor = zlib_executor + + @property + def body(self) -> Optional[Union[bytes, Payload]]: + return self._body + + @body.setter + def body(self, body: Any) -> None: + if body is None: + self._body = None + elif isinstance(body, (bytes, bytearray)): + self._body = body + else: + try: + self._body = body = payload.PAYLOAD_REGISTRY.get(body) + except payload.LookupError: + raise ValueError("Unsupported body type %r" % type(body)) + + headers = self._headers + + # set content-type + if hdrs.CONTENT_TYPE not in headers: + headers[hdrs.CONTENT_TYPE] = body.content_type + + # copy payload headers + if body.headers: + for key, value in body.headers.items(): + if key not in headers: + headers[key] = value + + self._compressed_body = None + + @property + def text(self) -> Optional[str]: + if self._body is None: + return None + # Note: When _body is a Payload (e.g. FilePayload), this may do blocking I/O + # This is generally safe as most common payloads (BytesPayload, StringPayload) + # don't do blocking I/O, but be careful with file-based payloads + return self._body.decode(self.charset or "utf-8") + + @text.setter + def text(self, text: str) -> None: + assert text is None or isinstance( + text, str + ), "text argument must be str (%r)" % type(text) + + if self.content_type == "application/octet-stream": + self.content_type = "text/plain" + if self.charset is None: + self.charset = "utf-8" + + self._body = text.encode(self.charset) + self._compressed_body = None + + @property + def content_length(self) -> Optional[int]: + if self._chunked: + return None + + if hdrs.CONTENT_LENGTH in self._headers: + return int(self._headers[hdrs.CONTENT_LENGTH]) + + if self._compressed_body is not None: + # Return length of the compressed body + return len(self._compressed_body) + elif isinstance(self._body, Payload): + # A payload without content length, or a compressed payload + return None + elif self._body is not None: + return len(self._body) + else: + return 0 + + @content_length.setter + def content_length(self, value: Optional[int]) -> None: + raise RuntimeError("Content length is set automatically") + + async def write_eof(self, data: bytes = b"") -> None: + if self._eof_sent: + return + if self._compressed_body is None: + body: Optional[Union[bytes, Payload]] = self._body + else: + body = self._compressed_body + assert not data, f"data arg is not supported, got {data!r}" + assert self._req is not None + assert self._payload_writer is not None + if body is None or self._must_be_empty_body: + await super().write_eof() + elif isinstance(self._body, Payload): + await self._body.write(self._payload_writer) + await self._body.close() + await super().write_eof() + else: + await super().write_eof(cast(bytes, body)) + + async def _start(self, request: "BaseRequest") -> AbstractStreamWriter: + if hdrs.CONTENT_LENGTH in self._headers: + if should_remove_content_length(request.method, self.status): + del self._headers[hdrs.CONTENT_LENGTH] + elif not self._chunked: + if isinstance(self._body, Payload): + if (size := self._body.size) is not None: + self._headers[hdrs.CONTENT_LENGTH] = str(size) + else: + body_len = len(self._body) if self._body else "0" + # https://www.rfc-editor.org/rfc/rfc9110.html#section-8.6-7 + if body_len != "0" or ( + self.status != 304 and request.method not in hdrs.METH_HEAD_ALL + ): + self._headers[hdrs.CONTENT_LENGTH] = str(body_len) + + return await super()._start(request) + + async def _do_start_compression(self, coding: ContentCoding) -> None: + if self._chunked or isinstance(self._body, Payload): + return await super()._do_start_compression(coding) + if coding is ContentCoding.identity: + return + # Instead of using _payload_writer.enable_compression, + # compress the whole body + compressor = ZLibCompressor( + encoding=coding.value, + max_sync_chunk_size=self._zlib_executor_size, + executor=self._zlib_executor, + ) + assert self._body is not None + if self._zlib_executor_size is None and len(self._body) > LARGE_BODY_SIZE: + warnings.warn( + "Synchronous compression of large response bodies " + f"({len(self._body)} bytes) might block the async event loop. " + "Consider providing a custom value to zlib_executor_size/" + "zlib_executor response properties or disabling compression on it." + ) + self._compressed_body = ( + await compressor.compress(self._body) + compressor.flush() + ) + self._headers[hdrs.CONTENT_ENCODING] = coding.value + self._headers[hdrs.CONTENT_LENGTH] = str(len(self._compressed_body)) + + +def json_response( + data: Any = sentinel, + *, + text: Optional[str] = None, + body: Optional[bytes] = None, + status: int = 200, + reason: Optional[str] = None, + headers: Optional[LooseHeaders] = None, + content_type: str = "application/json", + dumps: JSONEncoder = json.dumps, +) -> Response: + if data is not sentinel: + if text or body: + raise ValueError("only one of data, text, or body should be specified") + else: + text = dumps(data) + return Response( + text=text, + body=body, + status=status, + reason=reason, + headers=headers, + content_type=content_type, + ) diff --git a/lib/python3.1/site-packages/aiohttp/web_routedef.py b/lib/python3.1/site-packages/aiohttp/web_routedef.py new file mode 100644 index 0000000000000000000000000000000000000000..f51b6cd00815a4daeabf7ef269a3225b2b764503 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web_routedef.py @@ -0,0 +1,214 @@ +import abc +import os # noqa +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterator, + List, + Optional, + Sequence, + Type, + Union, + overload, +) + +import attr + +from . import hdrs +from .abc import AbstractView +from .typedefs import Handler, PathLike + +if TYPE_CHECKING: + from .web_request import Request + from .web_response import StreamResponse + from .web_urldispatcher import AbstractRoute, UrlDispatcher +else: + Request = StreamResponse = UrlDispatcher = AbstractRoute = None + + +__all__ = ( + "AbstractRouteDef", + "RouteDef", + "StaticDef", + "RouteTableDef", + "head", + "options", + "get", + "post", + "patch", + "put", + "delete", + "route", + "view", + "static", +) + + +class AbstractRouteDef(abc.ABC): + @abc.abstractmethod + def register(self, router: UrlDispatcher) -> List[AbstractRoute]: + pass # pragma: no cover + + +_HandlerType = Union[Type[AbstractView], Handler] + + +@attr.s(auto_attribs=True, frozen=True, repr=False, slots=True) +class RouteDef(AbstractRouteDef): + method: str + path: str + handler: _HandlerType + kwargs: Dict[str, Any] + + def __repr__(self) -> str: + info = [] + for name, value in sorted(self.kwargs.items()): + info.append(f", {name}={value!r}") + return " {handler.__name__!r}{info}>".format( + method=self.method, path=self.path, handler=self.handler, info="".join(info) + ) + + def register(self, router: UrlDispatcher) -> List[AbstractRoute]: + if self.method in hdrs.METH_ALL: + reg = getattr(router, "add_" + self.method.lower()) + return [reg(self.path, self.handler, **self.kwargs)] + else: + return [ + router.add_route(self.method, self.path, self.handler, **self.kwargs) + ] + + +@attr.s(auto_attribs=True, frozen=True, repr=False, slots=True) +class StaticDef(AbstractRouteDef): + prefix: str + path: PathLike + kwargs: Dict[str, Any] + + def __repr__(self) -> str: + info = [] + for name, value in sorted(self.kwargs.items()): + info.append(f", {name}={value!r}") + return " {path}{info}>".format( + prefix=self.prefix, path=self.path, info="".join(info) + ) + + def register(self, router: UrlDispatcher) -> List[AbstractRoute]: + resource = router.add_static(self.prefix, self.path, **self.kwargs) + routes = resource.get_info().get("routes", {}) + return list(routes.values()) + + +def route(method: str, path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return RouteDef(method, path, handler, kwargs) + + +def head(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_HEAD, path, handler, **kwargs) + + +def options(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_OPTIONS, path, handler, **kwargs) + + +def get( + path: str, + handler: _HandlerType, + *, + name: Optional[str] = None, + allow_head: bool = True, + **kwargs: Any, +) -> RouteDef: + return route( + hdrs.METH_GET, path, handler, name=name, allow_head=allow_head, **kwargs + ) + + +def post(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_POST, path, handler, **kwargs) + + +def put(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_PUT, path, handler, **kwargs) + + +def patch(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_PATCH, path, handler, **kwargs) + + +def delete(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef: + return route(hdrs.METH_DELETE, path, handler, **kwargs) + + +def view(path: str, handler: Type[AbstractView], **kwargs: Any) -> RouteDef: + return route(hdrs.METH_ANY, path, handler, **kwargs) + + +def static(prefix: str, path: PathLike, **kwargs: Any) -> StaticDef: + return StaticDef(prefix, path, kwargs) + + +_Deco = Callable[[_HandlerType], _HandlerType] + + +class RouteTableDef(Sequence[AbstractRouteDef]): + """Route definition table""" + + def __init__(self) -> None: + self._items: List[AbstractRouteDef] = [] + + def __repr__(self) -> str: + return f"" + + @overload + def __getitem__(self, index: int) -> AbstractRouteDef: ... + + @overload + def __getitem__(self, index: slice) -> List[AbstractRouteDef]: ... + + def __getitem__(self, index): # type: ignore[no-untyped-def] + return self._items[index] + + def __iter__(self) -> Iterator[AbstractRouteDef]: + return iter(self._items) + + def __len__(self) -> int: + return len(self._items) + + def __contains__(self, item: object) -> bool: + return item in self._items + + def route(self, method: str, path: str, **kwargs: Any) -> _Deco: + def inner(handler: _HandlerType) -> _HandlerType: + self._items.append(RouteDef(method, path, handler, kwargs)) + return handler + + return inner + + def head(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_HEAD, path, **kwargs) + + def get(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_GET, path, **kwargs) + + def post(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_POST, path, **kwargs) + + def put(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_PUT, path, **kwargs) + + def patch(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_PATCH, path, **kwargs) + + def delete(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_DELETE, path, **kwargs) + + def options(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_OPTIONS, path, **kwargs) + + def view(self, path: str, **kwargs: Any) -> _Deco: + return self.route(hdrs.METH_ANY, path, **kwargs) + + def static(self, prefix: str, path: PathLike, **kwargs: Any) -> None: + self._items.append(StaticDef(prefix, path, kwargs)) diff --git a/lib/python3.1/site-packages/aiohttp/web_runner.py b/lib/python3.1/site-packages/aiohttp/web_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..bcfec727c8419bbc6518085ecedde1f7de8992c9 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web_runner.py @@ -0,0 +1,399 @@ +import asyncio +import signal +import socket +import warnings +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, List, Optional, Set + +from yarl import URL + +from .typedefs import PathLike +from .web_app import Application +from .web_server import Server + +if TYPE_CHECKING: + from ssl import SSLContext +else: + try: + from ssl import SSLContext + except ImportError: # pragma: no cover + SSLContext = object # type: ignore[misc,assignment] + +__all__ = ( + "BaseSite", + "TCPSite", + "UnixSite", + "NamedPipeSite", + "SockSite", + "BaseRunner", + "AppRunner", + "ServerRunner", + "GracefulExit", +) + + +class GracefulExit(SystemExit): + code = 1 + + +def _raise_graceful_exit() -> None: + raise GracefulExit() + + +class BaseSite(ABC): + __slots__ = ("_runner", "_ssl_context", "_backlog", "_server") + + def __init__( + self, + runner: "BaseRunner", + *, + shutdown_timeout: float = 60.0, + ssl_context: Optional[SSLContext] = None, + backlog: int = 128, + ) -> None: + if runner.server is None: + raise RuntimeError("Call runner.setup() before making a site") + if shutdown_timeout != 60.0: + msg = "shutdown_timeout should be set on BaseRunner" + warnings.warn(msg, DeprecationWarning, stacklevel=2) + runner._shutdown_timeout = shutdown_timeout + self._runner = runner + self._ssl_context = ssl_context + self._backlog = backlog + self._server: Optional[asyncio.AbstractServer] = None + + @property + @abstractmethod + def name(self) -> str: + pass # pragma: no cover + + @abstractmethod + async def start(self) -> None: + self._runner._reg_site(self) + + async def stop(self) -> None: + self._runner._check_site(self) + if self._server is not None: # Maybe not started yet + self._server.close() + + self._runner._unreg_site(self) + + +class TCPSite(BaseSite): + __slots__ = ("_host", "_port", "_reuse_address", "_reuse_port") + + def __init__( + self, + runner: "BaseRunner", + host: Optional[str] = None, + port: Optional[int] = None, + *, + shutdown_timeout: float = 60.0, + ssl_context: Optional[SSLContext] = None, + backlog: int = 128, + reuse_address: Optional[bool] = None, + reuse_port: Optional[bool] = None, + ) -> None: + super().__init__( + runner, + shutdown_timeout=shutdown_timeout, + ssl_context=ssl_context, + backlog=backlog, + ) + self._host = host + if port is None: + port = 8443 if self._ssl_context else 8080 + self._port = port + self._reuse_address = reuse_address + self._reuse_port = reuse_port + + @property + def name(self) -> str: + scheme = "https" if self._ssl_context else "http" + host = "0.0.0.0" if not self._host else self._host + return str(URL.build(scheme=scheme, host=host, port=self._port)) + + async def start(self) -> None: + await super().start() + loop = asyncio.get_event_loop() + server = self._runner.server + assert server is not None + self._server = await loop.create_server( + server, + self._host, + self._port, + ssl=self._ssl_context, + backlog=self._backlog, + reuse_address=self._reuse_address, + reuse_port=self._reuse_port, + ) + + +class UnixSite(BaseSite): + __slots__ = ("_path",) + + def __init__( + self, + runner: "BaseRunner", + path: PathLike, + *, + shutdown_timeout: float = 60.0, + ssl_context: Optional[SSLContext] = None, + backlog: int = 128, + ) -> None: + super().__init__( + runner, + shutdown_timeout=shutdown_timeout, + ssl_context=ssl_context, + backlog=backlog, + ) + self._path = path + + @property + def name(self) -> str: + scheme = "https" if self._ssl_context else "http" + return f"{scheme}://unix:{self._path}:" + + async def start(self) -> None: + await super().start() + loop = asyncio.get_event_loop() + server = self._runner.server + assert server is not None + self._server = await loop.create_unix_server( + server, + self._path, + ssl=self._ssl_context, + backlog=self._backlog, + ) + + +class NamedPipeSite(BaseSite): + __slots__ = ("_path",) + + def __init__( + self, runner: "BaseRunner", path: str, *, shutdown_timeout: float = 60.0 + ) -> None: + loop = asyncio.get_event_loop() + if not isinstance( + loop, asyncio.ProactorEventLoop # type: ignore[attr-defined] + ): + raise RuntimeError( + "Named Pipes only available in proactor loop under windows" + ) + super().__init__(runner, shutdown_timeout=shutdown_timeout) + self._path = path + + @property + def name(self) -> str: + return self._path + + async def start(self) -> None: + await super().start() + loop = asyncio.get_event_loop() + server = self._runner.server + assert server is not None + _server = await loop.start_serving_pipe( # type: ignore[attr-defined] + server, self._path + ) + self._server = _server[0] + + +class SockSite(BaseSite): + __slots__ = ("_sock", "_name") + + def __init__( + self, + runner: "BaseRunner", + sock: socket.socket, + *, + shutdown_timeout: float = 60.0, + ssl_context: Optional[SSLContext] = None, + backlog: int = 128, + ) -> None: + super().__init__( + runner, + shutdown_timeout=shutdown_timeout, + ssl_context=ssl_context, + backlog=backlog, + ) + self._sock = sock + scheme = "https" if self._ssl_context else "http" + if hasattr(socket, "AF_UNIX") and sock.family == socket.AF_UNIX: + name = f"{scheme}://unix:{sock.getsockname()}:" + else: + host, port = sock.getsockname()[:2] + name = str(URL.build(scheme=scheme, host=host, port=port)) + self._name = name + + @property + def name(self) -> str: + return self._name + + async def start(self) -> None: + await super().start() + loop = asyncio.get_event_loop() + server = self._runner.server + assert server is not None + self._server = await loop.create_server( + server, sock=self._sock, ssl=self._ssl_context, backlog=self._backlog + ) + + +class BaseRunner(ABC): + __slots__ = ("_handle_signals", "_kwargs", "_server", "_sites", "_shutdown_timeout") + + def __init__( + self, + *, + handle_signals: bool = False, + shutdown_timeout: float = 60.0, + **kwargs: Any, + ) -> None: + self._handle_signals = handle_signals + self._kwargs = kwargs + self._server: Optional[Server] = None + self._sites: List[BaseSite] = [] + self._shutdown_timeout = shutdown_timeout + + @property + def server(self) -> Optional[Server]: + return self._server + + @property + def addresses(self) -> List[Any]: + ret: List[Any] = [] + for site in self._sites: + server = site._server + if server is not None: + sockets = server.sockets # type: ignore[attr-defined] + if sockets is not None: + for sock in sockets: + ret.append(sock.getsockname()) + return ret + + @property + def sites(self) -> Set[BaseSite]: + return set(self._sites) + + async def setup(self) -> None: + loop = asyncio.get_event_loop() + + if self._handle_signals: + try: + loop.add_signal_handler(signal.SIGINT, _raise_graceful_exit) + loop.add_signal_handler(signal.SIGTERM, _raise_graceful_exit) + except NotImplementedError: # pragma: no cover + # add_signal_handler is not implemented on Windows + pass + + self._server = await self._make_server() + + @abstractmethod + async def shutdown(self) -> None: + """Call any shutdown hooks to help server close gracefully.""" + + async def cleanup(self) -> None: + # The loop over sites is intentional, an exception on gather() + # leaves self._sites in unpredictable state. + # The loop guaranties that a site is either deleted on success or + # still present on failure + for site in list(self._sites): + await site.stop() + + if self._server: # If setup succeeded + # Yield to event loop to ensure incoming requests prior to stopping the sites + # have all started to be handled before we proceed to close idle connections. + await asyncio.sleep(0) + self._server.pre_shutdown() + await self.shutdown() + await self._server.shutdown(self._shutdown_timeout) + await self._cleanup_server() + + self._server = None + if self._handle_signals: + loop = asyncio.get_running_loop() + try: + loop.remove_signal_handler(signal.SIGINT) + loop.remove_signal_handler(signal.SIGTERM) + except NotImplementedError: # pragma: no cover + # remove_signal_handler is not implemented on Windows + pass + + @abstractmethod + async def _make_server(self) -> Server: + pass # pragma: no cover + + @abstractmethod + async def _cleanup_server(self) -> None: + pass # pragma: no cover + + def _reg_site(self, site: BaseSite) -> None: + if site in self._sites: + raise RuntimeError(f"Site {site} is already registered in runner {self}") + self._sites.append(site) + + def _check_site(self, site: BaseSite) -> None: + if site not in self._sites: + raise RuntimeError(f"Site {site} is not registered in runner {self}") + + def _unreg_site(self, site: BaseSite) -> None: + if site not in self._sites: + raise RuntimeError(f"Site {site} is not registered in runner {self}") + self._sites.remove(site) + + +class ServerRunner(BaseRunner): + """Low-level web server runner""" + + __slots__ = ("_web_server",) + + def __init__( + self, web_server: Server, *, handle_signals: bool = False, **kwargs: Any + ) -> None: + super().__init__(handle_signals=handle_signals, **kwargs) + self._web_server = web_server + + async def shutdown(self) -> None: + pass + + async def _make_server(self) -> Server: + return self._web_server + + async def _cleanup_server(self) -> None: + pass + + +class AppRunner(BaseRunner): + """Web Application runner""" + + __slots__ = ("_app",) + + def __init__( + self, app: Application, *, handle_signals: bool = False, **kwargs: Any + ) -> None: + super().__init__(handle_signals=handle_signals, **kwargs) + if not isinstance(app, Application): + raise TypeError( + "The first argument should be web.Application " + "instance, got {!r}".format(app) + ) + self._app = app + + @property + def app(self) -> Application: + return self._app + + async def shutdown(self) -> None: + await self._app.shutdown() + + async def _make_server(self) -> Server: + loop = asyncio.get_event_loop() + self._app._set_loop(loop) + self._app.on_startup.freeze() + await self._app.startup() + self._app.freeze() + + return self._app._make_handler(loop=loop, **self._kwargs) + + async def _cleanup_server(self) -> None: + await self._app.cleanup() diff --git a/lib/python3.1/site-packages/aiohttp/web_server.py b/lib/python3.1/site-packages/aiohttp/web_server.py new file mode 100644 index 0000000000000000000000000000000000000000..328aca1e405ef87e4df8a992c32eac092b4af8f0 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web_server.py @@ -0,0 +1,84 @@ +"""Low level HTTP server.""" + +import asyncio +from typing import Any, Awaitable, Callable, Dict, List, Optional # noqa + +from .abc import AbstractStreamWriter +from .http_parser import RawRequestMessage +from .streams import StreamReader +from .web_protocol import RequestHandler, _RequestFactory, _RequestHandler +from .web_request import BaseRequest + +__all__ = ("Server",) + + +class Server: + def __init__( + self, + handler: _RequestHandler, + *, + request_factory: Optional[_RequestFactory] = None, + handler_cancellation: bool = False, + loop: Optional[asyncio.AbstractEventLoop] = None, + **kwargs: Any, + ) -> None: + self._loop = loop or asyncio.get_running_loop() + self._connections: Dict[RequestHandler, asyncio.Transport] = {} + self._kwargs = kwargs + # requests_count is the number of requests being processed by the server + # for the lifetime of the server. + self.requests_count = 0 + self.request_handler = handler + self.request_factory = request_factory or self._make_request + self.handler_cancellation = handler_cancellation + + @property + def connections(self) -> List[RequestHandler]: + return list(self._connections.keys()) + + def connection_made( + self, handler: RequestHandler, transport: asyncio.Transport + ) -> None: + self._connections[handler] = transport + + def connection_lost( + self, handler: RequestHandler, exc: Optional[BaseException] = None + ) -> None: + if handler in self._connections: + if handler._task_handler: + handler._task_handler.add_done_callback( + lambda f: self._connections.pop(handler, None) + ) + else: + del self._connections[handler] + + def _make_request( + self, + message: RawRequestMessage, + payload: StreamReader, + protocol: RequestHandler, + writer: AbstractStreamWriter, + task: "asyncio.Task[None]", + ) -> BaseRequest: + return BaseRequest(message, payload, protocol, writer, task, self._loop) + + def pre_shutdown(self) -> None: + for conn in self._connections: + conn.close() + + async def shutdown(self, timeout: Optional[float] = None) -> None: + coros = (conn.shutdown(timeout) for conn in self._connections) + await asyncio.gather(*coros) + self._connections.clear() + + def __call__(self) -> RequestHandler: + try: + return RequestHandler(self, loop=self._loop, **self._kwargs) + except TypeError: + # Failsafe creation: remove all custom handler_args + kwargs = { + k: v + for k, v in self._kwargs.items() + if k in ["debug", "access_log_class"] + } + return RequestHandler(self, loop=self._loop, **kwargs) diff --git a/lib/python3.1/site-packages/aiohttp/web_urldispatcher.py b/lib/python3.1/site-packages/aiohttp/web_urldispatcher.py new file mode 100644 index 0000000000000000000000000000000000000000..28ae2518fec3a8b59e1e045ba01d6ff1bad0cd13 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web_urldispatcher.py @@ -0,0 +1,1303 @@ +import abc +import asyncio +import base64 +import functools +import hashlib +import html +import inspect +import keyword +import os +import re +import sys +import warnings +from functools import wraps +from pathlib import Path +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Container, + Dict, + Final, + Generator, + Iterable, + Iterator, + List, + Mapping, + NoReturn, + Optional, + Pattern, + Set, + Sized, + Tuple, + Type, + TypedDict, + Union, + cast, +) + +from yarl import URL, __version__ as yarl_version + +from . import hdrs +from .abc import AbstractMatchInfo, AbstractRouter, AbstractView +from .helpers import DEBUG +from .http import HttpVersion11 +from .typedefs import Handler, PathLike +from .web_exceptions import ( + HTTPException, + HTTPExpectationFailed, + HTTPForbidden, + HTTPMethodNotAllowed, + HTTPNotFound, +) +from .web_fileresponse import FileResponse +from .web_request import Request +from .web_response import Response, StreamResponse +from .web_routedef import AbstractRouteDef + +__all__ = ( + "UrlDispatcher", + "UrlMappingMatchInfo", + "AbstractResource", + "Resource", + "PlainResource", + "DynamicResource", + "AbstractRoute", + "ResourceRoute", + "StaticResource", + "View", +) + + +if TYPE_CHECKING: + from .web_app import Application + + BaseDict = Dict[str, str] +else: + BaseDict = dict + +CIRCULAR_SYMLINK_ERROR = ( + (OSError,) + if sys.version_info < (3, 10) and sys.platform.startswith("win32") + else (RuntimeError,) if sys.version_info < (3, 13) else () +) + +YARL_VERSION: Final[Tuple[int, ...]] = tuple(map(int, yarl_version.split(".")[:2])) + +HTTP_METHOD_RE: Final[Pattern[str]] = re.compile( + r"^[0-9A-Za-z!#\$%&'\*\+\-\.\^_`\|~]+$" +) +ROUTE_RE: Final[Pattern[str]] = re.compile( + r"(\{[_a-zA-Z][^{}]*(?:\{[^{}]*\}[^{}]*)*\})" +) +PATH_SEP: Final[str] = re.escape("/") + + +_ExpectHandler = Callable[[Request], Awaitable[Optional[StreamResponse]]] +_Resolve = Tuple[Optional["UrlMappingMatchInfo"], Set[str]] + +html_escape = functools.partial(html.escape, quote=True) + + +class _InfoDict(TypedDict, total=False): + path: str + + formatter: str + pattern: Pattern[str] + + directory: Path + prefix: str + routes: Mapping[str, "AbstractRoute"] + + app: "Application" + + domain: str + + rule: "AbstractRuleMatching" + + http_exception: HTTPException + + +class AbstractResource(Sized, Iterable["AbstractRoute"]): + def __init__(self, *, name: Optional[str] = None) -> None: + self._name = name + + @property + def name(self) -> Optional[str]: + return self._name + + @property + @abc.abstractmethod + def canonical(self) -> str: + """Exposes the resource's canonical path. + + For example '/foo/bar/{name}' + + """ + + @abc.abstractmethod # pragma: no branch + def url_for(self, **kwargs: str) -> URL: + """Construct url for resource with additional params.""" + + @abc.abstractmethod # pragma: no branch + async def resolve(self, request: Request) -> _Resolve: + """Resolve resource. + + Return (UrlMappingMatchInfo, allowed_methods) pair. + """ + + @abc.abstractmethod + def add_prefix(self, prefix: str) -> None: + """Add a prefix to processed URLs. + + Required for subapplications support. + """ + + @abc.abstractmethod + def get_info(self) -> _InfoDict: + """Return a dict with additional info useful for introspection""" + + def freeze(self) -> None: + pass + + @abc.abstractmethod + def raw_match(self, path: str) -> bool: + """Perform a raw match against path""" + + +class AbstractRoute(abc.ABC): + def __init__( + self, + method: str, + handler: Union[Handler, Type[AbstractView]], + *, + expect_handler: Optional[_ExpectHandler] = None, + resource: Optional[AbstractResource] = None, + ) -> None: + + if expect_handler is None: + expect_handler = _default_expect_handler + + assert inspect.iscoroutinefunction(expect_handler) or ( + sys.version_info < (3, 14) and asyncio.iscoroutinefunction(expect_handler) + ), f"Coroutine is expected, got {expect_handler!r}" + + method = method.upper() + if not HTTP_METHOD_RE.match(method): + raise ValueError(f"{method} is not allowed HTTP method") + + assert callable(handler), handler + if inspect.iscoroutinefunction(handler) or ( + sys.version_info < (3, 14) and asyncio.iscoroutinefunction(handler) + ): + pass + elif inspect.isgeneratorfunction(handler): + warnings.warn( + "Bare generators are deprecated, use @coroutine wrapper", + DeprecationWarning, + ) + elif isinstance(handler, type) and issubclass(handler, AbstractView): + pass + else: + warnings.warn( + "Bare functions are deprecated, use async ones", DeprecationWarning + ) + + @wraps(handler) + async def handler_wrapper(request: Request) -> StreamResponse: + result = old_handler(request) # type: ignore[call-arg] + if asyncio.iscoroutine(result): + result = await result + assert isinstance(result, StreamResponse) + return result + + old_handler = handler + handler = handler_wrapper + + self._method = method + self._handler = handler + self._expect_handler = expect_handler + self._resource = resource + + @property + def method(self) -> str: + return self._method + + @property + def handler(self) -> Handler: + return self._handler + + @property + @abc.abstractmethod + def name(self) -> Optional[str]: + """Optional route's name, always equals to resource's name.""" + + @property + def resource(self) -> Optional[AbstractResource]: + return self._resource + + @abc.abstractmethod + def get_info(self) -> _InfoDict: + """Return a dict with additional info useful for introspection""" + + @abc.abstractmethod # pragma: no branch + def url_for(self, *args: str, **kwargs: str) -> URL: + """Construct url for route with additional params.""" + + async def handle_expect_header(self, request: Request) -> Optional[StreamResponse]: + return await self._expect_handler(request) + + +class UrlMappingMatchInfo(BaseDict, AbstractMatchInfo): + + __slots__ = ("_route", "_apps", "_current_app", "_frozen") + + def __init__(self, match_dict: Dict[str, str], route: AbstractRoute) -> None: + super().__init__(match_dict) + self._route = route + self._apps: List[Application] = [] + self._current_app: Optional[Application] = None + self._frozen = False + + @property + def handler(self) -> Handler: + return self._route.handler + + @property + def route(self) -> AbstractRoute: + return self._route + + @property + def expect_handler(self) -> _ExpectHandler: + return self._route.handle_expect_header + + @property + def http_exception(self) -> Optional[HTTPException]: + return None + + def get_info(self) -> _InfoDict: # type: ignore[override] + return self._route.get_info() + + @property + def apps(self) -> Tuple["Application", ...]: + return tuple(self._apps) + + def add_app(self, app: "Application") -> None: + if self._frozen: + raise RuntimeError("Cannot change apps stack after .freeze() call") + if self._current_app is None: + self._current_app = app + self._apps.insert(0, app) + + @property + def current_app(self) -> "Application": + app = self._current_app + assert app is not None + return app + + @current_app.setter + def current_app(self, app: "Application") -> None: + if DEBUG: # pragma: no cover + if app not in self._apps: + raise RuntimeError( + "Expected one of the following apps {!r}, got {!r}".format( + self._apps, app + ) + ) + self._current_app = app + + def freeze(self) -> None: + self._frozen = True + + def __repr__(self) -> str: + return f"" + + +class MatchInfoError(UrlMappingMatchInfo): + + __slots__ = ("_exception",) + + def __init__(self, http_exception: HTTPException) -> None: + self._exception = http_exception + super().__init__({}, SystemRoute(self._exception)) + + @property + def http_exception(self) -> HTTPException: + return self._exception + + def __repr__(self) -> str: + return "".format( + self._exception.status, self._exception.reason + ) + + +async def _default_expect_handler(request: Request) -> None: + """Default handler for Expect header. + + Just send "100 Continue" to client. + raise HTTPExpectationFailed if value of header is not "100-continue" + """ + expect = request.headers.get(hdrs.EXPECT, "") + if request.version == HttpVersion11: + if expect.lower() == "100-continue": + await request.writer.write(b"HTTP/1.1 100 Continue\r\n\r\n") + # Reset output_size as we haven't started the main body yet. + request.writer.output_size = 0 + else: + raise HTTPExpectationFailed(text="Unknown Expect: %s" % expect) + + +class Resource(AbstractResource): + def __init__(self, *, name: Optional[str] = None) -> None: + super().__init__(name=name) + self._routes: Dict[str, ResourceRoute] = {} + self._any_route: Optional[ResourceRoute] = None + self._allowed_methods: Set[str] = set() + + def add_route( + self, + method: str, + handler: Union[Type[AbstractView], Handler], + *, + expect_handler: Optional[_ExpectHandler] = None, + ) -> "ResourceRoute": + if route := self._routes.get(method, self._any_route): + raise RuntimeError( + "Added route will never be executed, " + f"method {route.method} is already " + "registered" + ) + + route_obj = ResourceRoute(method, handler, self, expect_handler=expect_handler) + self.register_route(route_obj) + return route_obj + + def register_route(self, route: "ResourceRoute") -> None: + assert isinstance( + route, ResourceRoute + ), f"Instance of Route class is required, got {route!r}" + if route.method == hdrs.METH_ANY: + self._any_route = route + self._allowed_methods.add(route.method) + self._routes[route.method] = route + + async def resolve(self, request: Request) -> _Resolve: + if (match_dict := self._match(request.rel_url.path_safe)) is None: + return None, set() + if route := self._routes.get(request.method, self._any_route): + return UrlMappingMatchInfo(match_dict, route), self._allowed_methods + return None, self._allowed_methods + + @abc.abstractmethod + def _match(self, path: str) -> Optional[Dict[str, str]]: + pass # pragma: no cover + + def __len__(self) -> int: + return len(self._routes) + + def __iter__(self) -> Iterator["ResourceRoute"]: + return iter(self._routes.values()) + + # TODO: implement all abstract methods + + +class PlainResource(Resource): + def __init__(self, path: str, *, name: Optional[str] = None) -> None: + super().__init__(name=name) + assert not path or path.startswith("/") + self._path = path + + @property + def canonical(self) -> str: + return self._path + + def freeze(self) -> None: + if not self._path: + self._path = "/" + + def add_prefix(self, prefix: str) -> None: + assert prefix.startswith("/") + assert not prefix.endswith("/") + assert len(prefix) > 1 + self._path = prefix + self._path + + def _match(self, path: str) -> Optional[Dict[str, str]]: + # string comparison is about 10 times faster than regexp matching + if self._path == path: + return {} + return None + + def raw_match(self, path: str) -> bool: + return self._path == path + + def get_info(self) -> _InfoDict: + return {"path": self._path} + + def url_for(self) -> URL: # type: ignore[override] + return URL.build(path=self._path, encoded=True) + + def __repr__(self) -> str: + name = "'" + self.name + "' " if self.name is not None else "" + return f"" + + +class DynamicResource(Resource): + + DYN = re.compile(r"\{(?P[_a-zA-Z][_a-zA-Z0-9]*)\}") + DYN_WITH_RE = re.compile(r"\{(?P[_a-zA-Z][_a-zA-Z0-9]*):(?P.+)\}") + GOOD = r"[^{}/]+" + + def __init__(self, path: str, *, name: Optional[str] = None) -> None: + super().__init__(name=name) + self._orig_path = path + pattern = "" + formatter = "" + for part in ROUTE_RE.split(path): + match = self.DYN.fullmatch(part) + if match: + pattern += "(?P<{}>{})".format(match.group("var"), self.GOOD) + formatter += "{" + match.group("var") + "}" + continue + + match = self.DYN_WITH_RE.fullmatch(part) + if match: + pattern += "(?P<{var}>{re})".format(**match.groupdict()) + formatter += "{" + match.group("var") + "}" + continue + + if "{" in part or "}" in part: + raise ValueError(f"Invalid path '{path}'['{part}']") + + part = _requote_path(part) + formatter += part + pattern += re.escape(part) + + try: + compiled = re.compile(pattern) + except re.error as exc: + raise ValueError(f"Bad pattern '{pattern}': {exc}") from None + assert compiled.pattern.startswith(PATH_SEP) + assert formatter.startswith("/") + self._pattern = compiled + self._formatter = formatter + + @property + def canonical(self) -> str: + return self._formatter + + def add_prefix(self, prefix: str) -> None: + assert prefix.startswith("/") + assert not prefix.endswith("/") + assert len(prefix) > 1 + self._pattern = re.compile(re.escape(prefix) + self._pattern.pattern) + self._formatter = prefix + self._formatter + + def _match(self, path: str) -> Optional[Dict[str, str]]: + match = self._pattern.fullmatch(path) + if match is None: + return None + return { + key: _unquote_path_safe(value) for key, value in match.groupdict().items() + } + + def raw_match(self, path: str) -> bool: + return self._orig_path == path + + def get_info(self) -> _InfoDict: + return {"formatter": self._formatter, "pattern": self._pattern} + + def url_for(self, **parts: str) -> URL: + url = self._formatter.format_map({k: _quote_path(v) for k, v in parts.items()}) + return URL.build(path=url, encoded=True) + + def __repr__(self) -> str: + name = "'" + self.name + "' " if self.name is not None else "" + return "".format( + name=name, formatter=self._formatter + ) + + +class PrefixResource(AbstractResource): + def __init__(self, prefix: str, *, name: Optional[str] = None) -> None: + assert not prefix or prefix.startswith("/"), prefix + assert prefix in ("", "/") or not prefix.endswith("/"), prefix + super().__init__(name=name) + self._prefix = _requote_path(prefix) + self._prefix2 = self._prefix + "/" + + @property + def canonical(self) -> str: + return self._prefix + + def add_prefix(self, prefix: str) -> None: + assert prefix.startswith("/") + assert not prefix.endswith("/") + assert len(prefix) > 1 + self._prefix = prefix + self._prefix + self._prefix2 = self._prefix + "/" + + def raw_match(self, prefix: str) -> bool: + return False + + # TODO: impl missing abstract methods + + +class StaticResource(PrefixResource): + VERSION_KEY = "v" + + def __init__( + self, + prefix: str, + directory: PathLike, + *, + name: Optional[str] = None, + expect_handler: Optional[_ExpectHandler] = None, + chunk_size: int = 256 * 1024, + show_index: bool = False, + follow_symlinks: bool = False, + append_version: bool = False, + ) -> None: + super().__init__(prefix, name=name) + try: + directory = Path(directory).expanduser().resolve(strict=True) + except FileNotFoundError as error: + raise ValueError(f"'{directory}' does not exist") from error + if not directory.is_dir(): + raise ValueError(f"'{directory}' is not a directory") + self._directory = directory + self._show_index = show_index + self._chunk_size = chunk_size + self._follow_symlinks = follow_symlinks + self._expect_handler = expect_handler + self._append_version = append_version + + self._routes = { + "GET": ResourceRoute( + "GET", self._handle, self, expect_handler=expect_handler + ), + "HEAD": ResourceRoute( + "HEAD", self._handle, self, expect_handler=expect_handler + ), + } + self._allowed_methods = set(self._routes) + + def url_for( # type: ignore[override] + self, + *, + filename: PathLike, + append_version: Optional[bool] = None, + ) -> URL: + if append_version is None: + append_version = self._append_version + filename = str(filename).lstrip("/") + + url = URL.build(path=self._prefix, encoded=True) + # filename is not encoded + if YARL_VERSION < (1, 6): + url = url / filename.replace("%", "%25") + else: + url = url / filename + + if append_version: + unresolved_path = self._directory.joinpath(filename) + try: + if self._follow_symlinks: + normalized_path = Path(os.path.normpath(unresolved_path)) + normalized_path.relative_to(self._directory) + filepath = normalized_path.resolve() + else: + filepath = unresolved_path.resolve() + filepath.relative_to(self._directory) + except (ValueError, FileNotFoundError): + # ValueError for case when path point to symlink + # with follow_symlinks is False + return url # relatively safe + if filepath.is_file(): + # TODO cache file content + # with file watcher for cache invalidation + with filepath.open("rb") as f: + file_bytes = f.read() + h = self._get_file_hash(file_bytes) + url = url.with_query({self.VERSION_KEY: h}) + return url + return url + + @staticmethod + def _get_file_hash(byte_array: bytes) -> str: + m = hashlib.sha256() # todo sha256 can be configurable param + m.update(byte_array) + b64 = base64.urlsafe_b64encode(m.digest()) + return b64.decode("ascii") + + def get_info(self) -> _InfoDict: + return { + "directory": self._directory, + "prefix": self._prefix, + "routes": self._routes, + } + + def set_options_route(self, handler: Handler) -> None: + if "OPTIONS" in self._routes: + raise RuntimeError("OPTIONS route was set already") + self._routes["OPTIONS"] = ResourceRoute( + "OPTIONS", handler, self, expect_handler=self._expect_handler + ) + self._allowed_methods.add("OPTIONS") + + async def resolve(self, request: Request) -> _Resolve: + path = request.rel_url.path_safe + method = request.method + if not path.startswith(self._prefix2) and path != self._prefix: + return None, set() + + allowed_methods = self._allowed_methods + if method not in allowed_methods: + return None, allowed_methods + + match_dict = {"filename": _unquote_path_safe(path[len(self._prefix) + 1 :])} + return (UrlMappingMatchInfo(match_dict, self._routes[method]), allowed_methods) + + def __len__(self) -> int: + return len(self._routes) + + def __iter__(self) -> Iterator[AbstractRoute]: + return iter(self._routes.values()) + + async def _handle(self, request: Request) -> StreamResponse: + rel_url = request.match_info["filename"] + filename = Path(rel_url) + if filename.anchor: + # rel_url is an absolute name like + # /static/\\machine_name\c$ or /static/D:\path + # where the static dir is totally different + raise HTTPForbidden() + + unresolved_path = self._directory.joinpath(filename) + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + None, self._resolve_path_to_response, unresolved_path + ) + + def _resolve_path_to_response(self, unresolved_path: Path) -> StreamResponse: + """Take the unresolved path and query the file system to form a response.""" + # Check for access outside the root directory. For follow symlinks, URI + # cannot traverse out, but symlinks can. Otherwise, no access outside + # root is permitted. + try: + if self._follow_symlinks: + normalized_path = Path(os.path.normpath(unresolved_path)) + normalized_path.relative_to(self._directory) + file_path = normalized_path.resolve() + else: + file_path = unresolved_path.resolve() + file_path.relative_to(self._directory) + except (ValueError, *CIRCULAR_SYMLINK_ERROR) as error: + # ValueError is raised for the relative check. Circular symlinks + # raise here on resolving for python < 3.13. + raise HTTPNotFound() from error + + # if path is a directory, return the contents if permitted. Note the + # directory check will raise if a segment is not readable. + try: + if file_path.is_dir(): + if self._show_index: + return Response( + text=self._directory_as_html(file_path), + content_type="text/html", + ) + else: + raise HTTPForbidden() + except PermissionError as error: + raise HTTPForbidden() from error + + # Return the file response, which handles all other checks. + return FileResponse(file_path, chunk_size=self._chunk_size) + + def _directory_as_html(self, dir_path: Path) -> str: + """returns directory's index as html.""" + assert dir_path.is_dir() + + relative_path_to_dir = dir_path.relative_to(self._directory).as_posix() + index_of = f"Index of /{html_escape(relative_path_to_dir)}" + h1 = f"

{index_of}

" + + index_list = [] + dir_index = dir_path.iterdir() + for _file in sorted(dir_index): + # show file url as relative to static path + rel_path = _file.relative_to(self._directory).as_posix() + quoted_file_url = _quote_path(f"{self._prefix}/{rel_path}") + + # if file is a directory, add '/' to the end of the name + if _file.is_dir(): + file_name = f"{_file.name}/" + else: + file_name = _file.name + + index_list.append( + f'
  • {html_escape(file_name)}
    • ' + ) + ul = "
      \n{}\n
    ".format("\n".join(index_list)) + body = f"\n{h1}\n{ul}\n" + + head_str = f"\n{index_of}\n" + html = f"\n{head_str}\n{body}\n" + + return html + + def __repr__(self) -> str: + name = "'" + self.name + "'" if self.name is not None else "" + return " {directory!r}>".format( + name=name, path=self._prefix, directory=self._directory + ) + + +class PrefixedSubAppResource(PrefixResource): + def __init__(self, prefix: str, app: "Application") -> None: + super().__init__(prefix) + self._app = app + self._add_prefix_to_resources(prefix) + + def add_prefix(self, prefix: str) -> None: + super().add_prefix(prefix) + self._add_prefix_to_resources(prefix) + + def _add_prefix_to_resources(self, prefix: str) -> None: + router = self._app.router + for resource in router.resources(): + # Since the canonical path of a resource is about + # to change, we need to unindex it and then reindex + router.unindex_resource(resource) + resource.add_prefix(prefix) + router.index_resource(resource) + + def url_for(self, *args: str, **kwargs: str) -> URL: + raise RuntimeError(".url_for() is not supported by sub-application root") + + def get_info(self) -> _InfoDict: + return {"app": self._app, "prefix": self._prefix} + + async def resolve(self, request: Request) -> _Resolve: + match_info = await self._app.router.resolve(request) + match_info.add_app(self._app) + if isinstance(match_info.http_exception, HTTPMethodNotAllowed): + methods = match_info.http_exception.allowed_methods + else: + methods = set() + return match_info, methods + + def __len__(self) -> int: + return len(self._app.router.routes()) + + def __iter__(self) -> Iterator[AbstractRoute]: + return iter(self._app.router.routes()) + + def __repr__(self) -> str: + return " {app!r}>".format( + prefix=self._prefix, app=self._app + ) + + +class AbstractRuleMatching(abc.ABC): + @abc.abstractmethod # pragma: no branch + async def match(self, request: Request) -> bool: + """Return bool if the request satisfies the criteria""" + + @abc.abstractmethod # pragma: no branch + def get_info(self) -> _InfoDict: + """Return a dict with additional info useful for introspection""" + + @property + @abc.abstractmethod # pragma: no branch + def canonical(self) -> str: + """Return a str""" + + +class Domain(AbstractRuleMatching): + re_part = re.compile(r"(?!-)[a-z\d-]{1,63}(? None: + super().__init__() + self._domain = self.validation(domain) + + @property + def canonical(self) -> str: + return self._domain + + def validation(self, domain: str) -> str: + if not isinstance(domain, str): + raise TypeError("Domain must be str") + domain = domain.rstrip(".").lower() + if not domain: + raise ValueError("Domain cannot be empty") + elif "://" in domain: + raise ValueError("Scheme not supported") + url = URL("http://" + domain) + assert url.raw_host is not None + if not all(self.re_part.fullmatch(x) for x in url.raw_host.split(".")): + raise ValueError("Domain not valid") + if url.port == 80: + return url.raw_host + return f"{url.raw_host}:{url.port}" + + async def match(self, request: Request) -> bool: + host = request.headers.get(hdrs.HOST) + if not host: + return False + return self.match_domain(host) + + def match_domain(self, host: str) -> bool: + return host.lower() == self._domain + + def get_info(self) -> _InfoDict: + return {"domain": self._domain} + + +class MaskDomain(Domain): + re_part = re.compile(r"(?!-)[a-z\d\*-]{1,63}(? None: + super().__init__(domain) + mask = self._domain.replace(".", r"\.").replace("*", ".*") + self._mask = re.compile(mask) + + @property + def canonical(self) -> str: + return self._mask.pattern + + def match_domain(self, host: str) -> bool: + return self._mask.fullmatch(host) is not None + + +class MatchedSubAppResource(PrefixedSubAppResource): + def __init__(self, rule: AbstractRuleMatching, app: "Application") -> None: + AbstractResource.__init__(self) + self._prefix = "" + self._app = app + self._rule = rule + + @property + def canonical(self) -> str: + return self._rule.canonical + + def get_info(self) -> _InfoDict: + return {"app": self._app, "rule": self._rule} + + async def resolve(self, request: Request) -> _Resolve: + if not await self._rule.match(request): + return None, set() + match_info = await self._app.router.resolve(request) + match_info.add_app(self._app) + if isinstance(match_info.http_exception, HTTPMethodNotAllowed): + methods = match_info.http_exception.allowed_methods + else: + methods = set() + return match_info, methods + + def __repr__(self) -> str: + return f" {self._app!r}>" + + +class ResourceRoute(AbstractRoute): + """A route with resource""" + + def __init__( + self, + method: str, + handler: Union[Handler, Type[AbstractView]], + resource: AbstractResource, + *, + expect_handler: Optional[_ExpectHandler] = None, + ) -> None: + super().__init__( + method, handler, expect_handler=expect_handler, resource=resource + ) + + def __repr__(self) -> str: + return " {handler!r}".format( + method=self.method, resource=self._resource, handler=self.handler + ) + + @property + def name(self) -> Optional[str]: + if self._resource is None: + return None + return self._resource.name + + def url_for(self, *args: str, **kwargs: str) -> URL: + """Construct url for route with additional params.""" + assert self._resource is not None + return self._resource.url_for(*args, **kwargs) + + def get_info(self) -> _InfoDict: + assert self._resource is not None + return self._resource.get_info() + + +class SystemRoute(AbstractRoute): + def __init__(self, http_exception: HTTPException) -> None: + super().__init__(hdrs.METH_ANY, self._handle) + self._http_exception = http_exception + + def url_for(self, *args: str, **kwargs: str) -> URL: + raise RuntimeError(".url_for() is not allowed for SystemRoute") + + @property + def name(self) -> Optional[str]: + return None + + def get_info(self) -> _InfoDict: + return {"http_exception": self._http_exception} + + async def _handle(self, request: Request) -> StreamResponse: + raise self._http_exception + + @property + def status(self) -> int: + return self._http_exception.status + + @property + def reason(self) -> str: + return self._http_exception.reason + + def __repr__(self) -> str: + return "".format(self=self) + + +class View(AbstractView): + async def _iter(self) -> StreamResponse: + if self.request.method not in hdrs.METH_ALL: + self._raise_allowed_methods() + method: Optional[Callable[[], Awaitable[StreamResponse]]] + method = getattr(self, self.request.method.lower(), None) + if method is None: + self._raise_allowed_methods() + ret = await method() + assert isinstance(ret, StreamResponse) + return ret + + def __await__(self) -> Generator[Any, None, StreamResponse]: + return self._iter().__await__() + + def _raise_allowed_methods(self) -> NoReturn: + allowed_methods = {m for m in hdrs.METH_ALL if hasattr(self, m.lower())} + raise HTTPMethodNotAllowed(self.request.method, allowed_methods) + + +class ResourcesView(Sized, Iterable[AbstractResource], Container[AbstractResource]): + def __init__(self, resources: List[AbstractResource]) -> None: + self._resources = resources + + def __len__(self) -> int: + return len(self._resources) + + def __iter__(self) -> Iterator[AbstractResource]: + yield from self._resources + + def __contains__(self, resource: object) -> bool: + return resource in self._resources + + +class RoutesView(Sized, Iterable[AbstractRoute], Container[AbstractRoute]): + def __init__(self, resources: List[AbstractResource]): + self._routes: List[AbstractRoute] = [] + for resource in resources: + for route in resource: + self._routes.append(route) + + def __len__(self) -> int: + return len(self._routes) + + def __iter__(self) -> Iterator[AbstractRoute]: + yield from self._routes + + def __contains__(self, route: object) -> bool: + return route in self._routes + + +class UrlDispatcher(AbstractRouter, Mapping[str, AbstractResource]): + + NAME_SPLIT_RE = re.compile(r"[.:-]") + + def __init__(self) -> None: + super().__init__() + self._resources: List[AbstractResource] = [] + self._named_resources: Dict[str, AbstractResource] = {} + self._resource_index: dict[str, list[AbstractResource]] = {} + self._matched_sub_app_resources: List[MatchedSubAppResource] = [] + + async def resolve(self, request: Request) -> UrlMappingMatchInfo: + resource_index = self._resource_index + allowed_methods: Set[str] = set() + + # Walk the url parts looking for candidates. We walk the url backwards + # to ensure the most explicit match is found first. If there are multiple + # candidates for a given url part because there are multiple resources + # registered for the same canonical path, we resolve them in a linear + # fashion to ensure registration order is respected. + url_part = request.rel_url.path_safe + while url_part: + for candidate in resource_index.get(url_part, ()): + match_dict, allowed = await candidate.resolve(request) + if match_dict is not None: + return match_dict + else: + allowed_methods |= allowed + if url_part == "/": + break + url_part = url_part.rpartition("/")[0] or "/" + + # + # We didn't find any candidates, so we'll try the matched sub-app + # resources which we have to walk in a linear fashion because they + # have regex/wildcard match rules and we cannot index them. + # + # For most cases we do not expect there to be many of these since + # currently they are only added by `add_domain` + # + for resource in self._matched_sub_app_resources: + match_dict, allowed = await resource.resolve(request) + if match_dict is not None: + return match_dict + else: + allowed_methods |= allowed + + if allowed_methods: + return MatchInfoError(HTTPMethodNotAllowed(request.method, allowed_methods)) + + return MatchInfoError(HTTPNotFound()) + + def __iter__(self) -> Iterator[str]: + return iter(self._named_resources) + + def __len__(self) -> int: + return len(self._named_resources) + + def __contains__(self, resource: object) -> bool: + return resource in self._named_resources + + def __getitem__(self, name: str) -> AbstractResource: + return self._named_resources[name] + + def resources(self) -> ResourcesView: + return ResourcesView(self._resources) + + def routes(self) -> RoutesView: + return RoutesView(self._resources) + + def named_resources(self) -> Mapping[str, AbstractResource]: + return MappingProxyType(self._named_resources) + + def register_resource(self, resource: AbstractResource) -> None: + assert isinstance( + resource, AbstractResource + ), f"Instance of AbstractResource class is required, got {resource!r}" + if self.frozen: + raise RuntimeError("Cannot register a resource into frozen router.") + + name = resource.name + + if name is not None: + parts = self.NAME_SPLIT_RE.split(name) + for part in parts: + if keyword.iskeyword(part): + raise ValueError( + f"Incorrect route name {name!r}, " + "python keywords cannot be used " + "for route name" + ) + if not part.isidentifier(): + raise ValueError( + "Incorrect route name {!r}, " + "the name should be a sequence of " + "python identifiers separated " + "by dash, dot or column".format(name) + ) + if name in self._named_resources: + raise ValueError( + "Duplicate {!r}, " + "already handled by {!r}".format(name, self._named_resources[name]) + ) + self._named_resources[name] = resource + self._resources.append(resource) + + if isinstance(resource, MatchedSubAppResource): + # We cannot index match sub-app resources because they have match rules + self._matched_sub_app_resources.append(resource) + else: + self.index_resource(resource) + + def _get_resource_index_key(self, resource: AbstractResource) -> str: + """Return a key to index the resource in the resource index.""" + if "{" in (index_key := resource.canonical): + # strip at the first { to allow for variables, and than + # rpartition at / to allow for variable parts in the path + # For example if the canonical path is `/core/locations{tail:.*}` + # the index key will be `/core` since index is based on the + # url parts split by `/` + index_key = index_key.partition("{")[0].rpartition("/")[0] + return index_key.rstrip("/") or "/" + + def index_resource(self, resource: AbstractResource) -> None: + """Add a resource to the resource index.""" + resource_key = self._get_resource_index_key(resource) + # There may be multiple resources for a canonical path + # so we keep them in a list to ensure that registration + # order is respected. + self._resource_index.setdefault(resource_key, []).append(resource) + + def unindex_resource(self, resource: AbstractResource) -> None: + """Remove a resource from the resource index.""" + resource_key = self._get_resource_index_key(resource) + self._resource_index[resource_key].remove(resource) + + def add_resource(self, path: str, *, name: Optional[str] = None) -> Resource: + if path and not path.startswith("/"): + raise ValueError("path should be started with / or be empty") + # Reuse last added resource if path and name are the same + if self._resources: + resource = self._resources[-1] + if resource.name == name and resource.raw_match(path): + return cast(Resource, resource) + if not ("{" in path or "}" in path or ROUTE_RE.search(path)): + resource = PlainResource(path, name=name) + self.register_resource(resource) + return resource + resource = DynamicResource(path, name=name) + self.register_resource(resource) + return resource + + def add_route( + self, + method: str, + path: str, + handler: Union[Handler, Type[AbstractView]], + *, + name: Optional[str] = None, + expect_handler: Optional[_ExpectHandler] = None, + ) -> AbstractRoute: + resource = self.add_resource(path, name=name) + return resource.add_route(method, handler, expect_handler=expect_handler) + + def add_static( + self, + prefix: str, + path: PathLike, + *, + name: Optional[str] = None, + expect_handler: Optional[_ExpectHandler] = None, + chunk_size: int = 256 * 1024, + show_index: bool = False, + follow_symlinks: bool = False, + append_version: bool = False, + ) -> AbstractResource: + """Add static files view. + + prefix - url prefix + path - folder with files + + """ + assert prefix.startswith("/") + if prefix.endswith("/"): + prefix = prefix[:-1] + resource = StaticResource( + prefix, + path, + name=name, + expect_handler=expect_handler, + chunk_size=chunk_size, + show_index=show_index, + follow_symlinks=follow_symlinks, + append_version=append_version, + ) + self.register_resource(resource) + return resource + + def add_head(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method HEAD.""" + return self.add_route(hdrs.METH_HEAD, path, handler, **kwargs) + + def add_options(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method OPTIONS.""" + return self.add_route(hdrs.METH_OPTIONS, path, handler, **kwargs) + + def add_get( + self, + path: str, + handler: Handler, + *, + name: Optional[str] = None, + allow_head: bool = True, + **kwargs: Any, + ) -> AbstractRoute: + """Shortcut for add_route with method GET. + + If allow_head is true, another + route is added allowing head requests to the same endpoint. + """ + resource = self.add_resource(path, name=name) + if allow_head: + resource.add_route(hdrs.METH_HEAD, handler, **kwargs) + return resource.add_route(hdrs.METH_GET, handler, **kwargs) + + def add_post(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method POST.""" + return self.add_route(hdrs.METH_POST, path, handler, **kwargs) + + def add_put(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method PUT.""" + return self.add_route(hdrs.METH_PUT, path, handler, **kwargs) + + def add_patch(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method PATCH.""" + return self.add_route(hdrs.METH_PATCH, path, handler, **kwargs) + + def add_delete(self, path: str, handler: Handler, **kwargs: Any) -> AbstractRoute: + """Shortcut for add_route with method DELETE.""" + return self.add_route(hdrs.METH_DELETE, path, handler, **kwargs) + + def add_view( + self, path: str, handler: Type[AbstractView], **kwargs: Any + ) -> AbstractRoute: + """Shortcut for add_route with ANY methods for a class-based view.""" + return self.add_route(hdrs.METH_ANY, path, handler, **kwargs) + + def freeze(self) -> None: + super().freeze() + for resource in self._resources: + resource.freeze() + + def add_routes(self, routes: Iterable[AbstractRouteDef]) -> List[AbstractRoute]: + """Append routes to route table. + + Parameter should be a sequence of RouteDef objects. + + Returns a list of registered AbstractRoute instances. + """ + registered_routes = [] + for route_def in routes: + registered_routes.extend(route_def.register(self)) + return registered_routes + + +def _quote_path(value: str) -> str: + if YARL_VERSION < (1, 6): + value = value.replace("%", "%25") + return URL.build(path=value, encoded=False).raw_path + + +def _unquote_path_safe(value: str) -> str: + if "%" not in value: + return value + return value.replace("%2F", "/").replace("%25", "%") + + +def _requote_path(value: str) -> str: + # Quote non-ascii characters and other characters which must be quoted, + # but preserve existing %-sequences. + result = _quote_path(value) + if "%" in value: + result = result.replace("%25", "%") + return result diff --git a/lib/python3.1/site-packages/aiohttp/web_ws.py b/lib/python3.1/site-packages/aiohttp/web_ws.py new file mode 100644 index 0000000000000000000000000000000000000000..575f9a3dc8507d1e6b766333c9daec389313febd --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/web_ws.py @@ -0,0 +1,631 @@ +import asyncio +import base64 +import binascii +import hashlib +import json +import sys +from typing import Any, Final, Iterable, Optional, Tuple, Union, cast + +import attr +from multidict import CIMultiDict + +from . import hdrs +from ._websocket.reader import WebSocketDataQueue +from ._websocket.writer import DEFAULT_LIMIT +from .abc import AbstractStreamWriter +from .client_exceptions import WSMessageTypeError +from .helpers import calculate_timeout_when, set_exception, set_result +from .http import ( + WS_CLOSED_MESSAGE, + WS_CLOSING_MESSAGE, + WS_KEY, + WebSocketError, + WebSocketReader, + WebSocketWriter, + WSCloseCode, + WSMessage, + WSMsgType as WSMsgType, + ws_ext_gen, + ws_ext_parse, +) +from .http_websocket import _INTERNAL_RECEIVE_TYPES +from .log import ws_logger +from .streams import EofStream +from .typedefs import JSONDecoder, JSONEncoder +from .web_exceptions import HTTPBadRequest, HTTPException +from .web_request import BaseRequest +from .web_response import StreamResponse + +if sys.version_info >= (3, 11): + import asyncio as async_timeout +else: + import async_timeout + +__all__ = ( + "WebSocketResponse", + "WebSocketReady", + "WSMsgType", +) + +THRESHOLD_CONNLOST_ACCESS: Final[int] = 5 + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class WebSocketReady: + ok: bool + protocol: Optional[str] + + def __bool__(self) -> bool: + return self.ok + + +class WebSocketResponse(StreamResponse): + + _length_check: bool = False + _ws_protocol: Optional[str] = None + _writer: Optional[WebSocketWriter] = None + _reader: Optional[WebSocketDataQueue] = None + _closed: bool = False + _closing: bool = False + _conn_lost: int = 0 + _close_code: Optional[int] = None + _loop: Optional[asyncio.AbstractEventLoop] = None + _waiting: bool = False + _close_wait: Optional[asyncio.Future[None]] = None + _exception: Optional[BaseException] = None + _heartbeat_when: float = 0.0 + _heartbeat_cb: Optional[asyncio.TimerHandle] = None + _pong_response_cb: Optional[asyncio.TimerHandle] = None + _ping_task: Optional[asyncio.Task[None]] = None + + def __init__( + self, + *, + timeout: float = 10.0, + receive_timeout: Optional[float] = None, + autoclose: bool = True, + autoping: bool = True, + heartbeat: Optional[float] = None, + protocols: Iterable[str] = (), + compress: bool = True, + max_msg_size: int = 4 * 1024 * 1024, + writer_limit: int = DEFAULT_LIMIT, + ) -> None: + super().__init__(status=101) + self._protocols = protocols + self._timeout = timeout + self._receive_timeout = receive_timeout + self._autoclose = autoclose + self._autoping = autoping + self._heartbeat = heartbeat + if heartbeat is not None: + self._pong_heartbeat = heartbeat / 2.0 + self._compress: Union[bool, int] = compress + self._max_msg_size = max_msg_size + self._writer_limit = writer_limit + + def _cancel_heartbeat(self) -> None: + self._cancel_pong_response_cb() + if self._heartbeat_cb is not None: + self._heartbeat_cb.cancel() + self._heartbeat_cb = None + if self._ping_task is not None: + self._ping_task.cancel() + self._ping_task = None + + def _cancel_pong_response_cb(self) -> None: + if self._pong_response_cb is not None: + self._pong_response_cb.cancel() + self._pong_response_cb = None + + def _reset_heartbeat(self) -> None: + if self._heartbeat is None: + return + self._cancel_pong_response_cb() + req = self._req + timeout_ceil_threshold = ( + req._protocol._timeout_ceil_threshold if req is not None else 5 + ) + loop = self._loop + assert loop is not None + now = loop.time() + when = calculate_timeout_when(now, self._heartbeat, timeout_ceil_threshold) + self._heartbeat_when = when + if self._heartbeat_cb is None: + # We do not cancel the previous heartbeat_cb here because + # it generates a significant amount of TimerHandle churn + # which causes asyncio to rebuild the heap frequently. + # Instead _send_heartbeat() will reschedule the next + # heartbeat if it fires too early. + self._heartbeat_cb = loop.call_at(when, self._send_heartbeat) + + def _send_heartbeat(self) -> None: + self._heartbeat_cb = None + loop = self._loop + assert loop is not None and self._writer is not None + now = loop.time() + if now < self._heartbeat_when: + # Heartbeat fired too early, reschedule + self._heartbeat_cb = loop.call_at( + self._heartbeat_when, self._send_heartbeat + ) + return + + req = self._req + timeout_ceil_threshold = ( + req._protocol._timeout_ceil_threshold if req is not None else 5 + ) + when = calculate_timeout_when(now, self._pong_heartbeat, timeout_ceil_threshold) + self._cancel_pong_response_cb() + self._pong_response_cb = loop.call_at(when, self._pong_not_received) + + coro = self._writer.send_frame(b"", WSMsgType.PING) + if sys.version_info >= (3, 12): + # Optimization for Python 3.12, try to send the ping + # immediately to avoid having to schedule + # the task on the event loop. + ping_task = asyncio.Task(coro, loop=loop, eager_start=True) + else: + ping_task = loop.create_task(coro) + + if not ping_task.done(): + self._ping_task = ping_task + ping_task.add_done_callback(self._ping_task_done) + else: + self._ping_task_done(ping_task) + + def _ping_task_done(self, task: "asyncio.Task[None]") -> None: + """Callback for when the ping task completes.""" + if not task.cancelled() and (exc := task.exception()): + self._handle_ping_pong_exception(exc) + self._ping_task = None + + def _pong_not_received(self) -> None: + if self._req is not None and self._req.transport is not None: + self._handle_ping_pong_exception( + asyncio.TimeoutError( + f"No PONG received after {self._pong_heartbeat} seconds" + ) + ) + + def _handle_ping_pong_exception(self, exc: BaseException) -> None: + """Handle exceptions raised during ping/pong processing.""" + if self._closed: + return + self._set_closed() + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + self._exception = exc + if self._waiting and not self._closing and self._reader is not None: + self._reader.feed_data(WSMessage(WSMsgType.ERROR, exc, None), 0) + + def _set_closed(self) -> None: + """Set the connection to closed. + + Cancel any heartbeat timers and set the closed flag. + """ + self._closed = True + self._cancel_heartbeat() + + async def prepare(self, request: BaseRequest) -> AbstractStreamWriter: + # make pre-check to don't hide it by do_handshake() exceptions + if self._payload_writer is not None: + return self._payload_writer + + protocol, writer = self._pre_start(request) + payload_writer = await super().prepare(request) + assert payload_writer is not None + self._post_start(request, protocol, writer) + await payload_writer.drain() + return payload_writer + + def _handshake( + self, request: BaseRequest + ) -> Tuple["CIMultiDict[str]", Optional[str], int, bool]: + headers = request.headers + if "websocket" != headers.get(hdrs.UPGRADE, "").lower().strip(): + raise HTTPBadRequest( + text=( + "No WebSocket UPGRADE hdr: {}\n Can " + '"Upgrade" only to "WebSocket".' + ).format(headers.get(hdrs.UPGRADE)) + ) + + if "upgrade" not in headers.get(hdrs.CONNECTION, "").lower(): + raise HTTPBadRequest( + text="No CONNECTION upgrade hdr: {}".format( + headers.get(hdrs.CONNECTION) + ) + ) + + # find common sub-protocol between client and server + protocol: Optional[str] = None + if hdrs.SEC_WEBSOCKET_PROTOCOL in headers: + req_protocols = [ + str(proto.strip()) + for proto in headers[hdrs.SEC_WEBSOCKET_PROTOCOL].split(",") + ] + + for proto in req_protocols: + if proto in self._protocols: + protocol = proto + break + else: + # No overlap found: Return no protocol as per spec + ws_logger.warning( + "%s: Client protocols %r don’t overlap server-known ones %r", + request.remote, + req_protocols, + self._protocols, + ) + + # check supported version + version = headers.get(hdrs.SEC_WEBSOCKET_VERSION, "") + if version not in ("13", "8", "7"): + raise HTTPBadRequest(text=f"Unsupported version: {version}") + + # check client handshake for validity + key = headers.get(hdrs.SEC_WEBSOCKET_KEY) + try: + if not key or len(base64.b64decode(key)) != 16: + raise HTTPBadRequest(text=f"Handshake error: {key!r}") + except binascii.Error: + raise HTTPBadRequest(text=f"Handshake error: {key!r}") from None + + accept_val = base64.b64encode( + hashlib.sha1(key.encode() + WS_KEY).digest() + ).decode() + response_headers = CIMultiDict( + { + hdrs.UPGRADE: "websocket", + hdrs.CONNECTION: "upgrade", + hdrs.SEC_WEBSOCKET_ACCEPT: accept_val, + } + ) + + notakeover = False + compress = 0 + if self._compress: + extensions = headers.get(hdrs.SEC_WEBSOCKET_EXTENSIONS) + # Server side always get return with no exception. + # If something happened, just drop compress extension + compress, notakeover = ws_ext_parse(extensions, isserver=True) + if compress: + enabledext = ws_ext_gen( + compress=compress, isserver=True, server_notakeover=notakeover + ) + response_headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = enabledext + + if protocol: + response_headers[hdrs.SEC_WEBSOCKET_PROTOCOL] = protocol + return ( + response_headers, + protocol, + compress, + notakeover, + ) + + def _pre_start(self, request: BaseRequest) -> Tuple[Optional[str], WebSocketWriter]: + self._loop = request._loop + + headers, protocol, compress, notakeover = self._handshake(request) + + self.set_status(101) + self.headers.update(headers) + self.force_close() + self._compress = compress + transport = request._protocol.transport + assert transport is not None + writer = WebSocketWriter( + request._protocol, + transport, + compress=compress, + notakeover=notakeover, + limit=self._writer_limit, + ) + + return protocol, writer + + def _post_start( + self, request: BaseRequest, protocol: Optional[str], writer: WebSocketWriter + ) -> None: + self._ws_protocol = protocol + self._writer = writer + + self._reset_heartbeat() + + loop = self._loop + assert loop is not None + self._reader = WebSocketDataQueue(request._protocol, 2**16, loop=loop) + request.protocol.set_parser( + WebSocketReader( + self._reader, self._max_msg_size, compress=bool(self._compress) + ) + ) + # disable HTTP keepalive for WebSocket + request.protocol.keep_alive(False) + + def can_prepare(self, request: BaseRequest) -> WebSocketReady: + if self._writer is not None: + raise RuntimeError("Already started") + try: + _, protocol, _, _ = self._handshake(request) + except HTTPException: + return WebSocketReady(False, None) + else: + return WebSocketReady(True, protocol) + + @property + def prepared(self) -> bool: + return self._writer is not None + + @property + def closed(self) -> bool: + return self._closed + + @property + def close_code(self) -> Optional[int]: + return self._close_code + + @property + def ws_protocol(self) -> Optional[str]: + return self._ws_protocol + + @property + def compress(self) -> Union[int, bool]: + return self._compress + + def get_extra_info(self, name: str, default: Any = None) -> Any: + """Get optional transport information. + + If no value associated with ``name`` is found, ``default`` is returned. + """ + writer = self._writer + if writer is None: + return default + transport = writer.transport + if transport is None: + return default + return transport.get_extra_info(name, default) + + def exception(self) -> Optional[BaseException]: + return self._exception + + async def ping(self, message: bytes = b"") -> None: + if self._writer is None: + raise RuntimeError("Call .prepare() first") + await self._writer.send_frame(message, WSMsgType.PING) + + async def pong(self, message: bytes = b"") -> None: + # unsolicited pong + if self._writer is None: + raise RuntimeError("Call .prepare() first") + await self._writer.send_frame(message, WSMsgType.PONG) + + async def send_frame( + self, message: bytes, opcode: WSMsgType, compress: Optional[int] = None + ) -> None: + """Send a frame over the websocket.""" + if self._writer is None: + raise RuntimeError("Call .prepare() first") + await self._writer.send_frame(message, opcode, compress) + + async def send_str(self, data: str, compress: Optional[int] = None) -> None: + if self._writer is None: + raise RuntimeError("Call .prepare() first") + if not isinstance(data, str): + raise TypeError("data argument must be str (%r)" % type(data)) + await self._writer.send_frame( + data.encode("utf-8"), WSMsgType.TEXT, compress=compress + ) + + async def send_bytes(self, data: bytes, compress: Optional[int] = None) -> None: + if self._writer is None: + raise RuntimeError("Call .prepare() first") + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError("data argument must be byte-ish (%r)" % type(data)) + await self._writer.send_frame(data, WSMsgType.BINARY, compress=compress) + + async def send_json( + self, + data: Any, + compress: Optional[int] = None, + *, + dumps: JSONEncoder = json.dumps, + ) -> None: + await self.send_str(dumps(data), compress=compress) + + async def write_eof(self) -> None: # type: ignore[override] + if self._eof_sent: + return + if self._payload_writer is None: + raise RuntimeError("Response has not been started") + + await self.close() + self._eof_sent = True + + async def close( + self, *, code: int = WSCloseCode.OK, message: bytes = b"", drain: bool = True + ) -> bool: + """Close websocket connection.""" + if self._writer is None: + raise RuntimeError("Call .prepare() first") + + if self._closed: + return False + self._set_closed() + + try: + await self._writer.close(code, message) + writer = self._payload_writer + assert writer is not None + if drain: + await writer.drain() + except (asyncio.CancelledError, asyncio.TimeoutError): + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + raise + except Exception as exc: + self._exception = exc + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + return True + + reader = self._reader + assert reader is not None + # we need to break `receive()` cycle before we can call + # `reader.read()` as `close()` may be called from different task + if self._waiting: + assert self._loop is not None + assert self._close_wait is None + self._close_wait = self._loop.create_future() + reader.feed_data(WS_CLOSING_MESSAGE, 0) + await self._close_wait + + if self._closing: + self._close_transport() + return True + + try: + async with async_timeout.timeout(self._timeout): + while True: + msg = await reader.read() + if msg.type is WSMsgType.CLOSE: + self._set_code_close_transport(msg.data) + return True + except asyncio.CancelledError: + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + raise + except Exception as exc: + self._exception = exc + self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE) + return True + + def _set_closing(self, code: WSCloseCode) -> None: + """Set the close code and mark the connection as closing.""" + self._closing = True + self._close_code = code + self._cancel_heartbeat() + + def _set_code_close_transport(self, code: WSCloseCode) -> None: + """Set the close code and close the transport.""" + self._close_code = code + self._close_transport() + + def _close_transport(self) -> None: + """Close the transport.""" + if self._req is not None and self._req.transport is not None: + self._req.transport.close() + + async def receive(self, timeout: Optional[float] = None) -> WSMessage: + if self._reader is None: + raise RuntimeError("Call .prepare() first") + + receive_timeout = timeout or self._receive_timeout + while True: + if self._waiting: + raise RuntimeError("Concurrent call to receive() is not allowed") + + if self._closed: + self._conn_lost += 1 + if self._conn_lost >= THRESHOLD_CONNLOST_ACCESS: + raise RuntimeError("WebSocket connection is closed.") + return WS_CLOSED_MESSAGE + elif self._closing: + return WS_CLOSING_MESSAGE + + try: + self._waiting = True + try: + if receive_timeout: + # Entering the context manager and creating + # Timeout() object can take almost 50% of the + # run time in this loop so we avoid it if + # there is no read timeout. + async with async_timeout.timeout(receive_timeout): + msg = await self._reader.read() + else: + msg = await self._reader.read() + self._reset_heartbeat() + finally: + self._waiting = False + if self._close_wait: + set_result(self._close_wait, None) + except asyncio.TimeoutError: + raise + except EofStream: + self._close_code = WSCloseCode.OK + await self.close() + return WSMessage(WSMsgType.CLOSED, None, None) + except WebSocketError as exc: + self._close_code = exc.code + await self.close(code=exc.code) + return WSMessage(WSMsgType.ERROR, exc, None) + except Exception as exc: + self._exception = exc + self._set_closing(WSCloseCode.ABNORMAL_CLOSURE) + await self.close() + return WSMessage(WSMsgType.ERROR, exc, None) + + if msg.type not in _INTERNAL_RECEIVE_TYPES: + # If its not a close/closing/ping/pong message + # we can return it immediately + return msg + + if msg.type is WSMsgType.CLOSE: + self._set_closing(msg.data) + # Could be closed while awaiting reader. + if not self._closed and self._autoclose: + # The client is likely going to close the + # connection out from under us so we do not + # want to drain any pending writes as it will + # likely result writing to a broken pipe. + await self.close(drain=False) + elif msg.type is WSMsgType.CLOSING: + self._set_closing(WSCloseCode.OK) + elif msg.type is WSMsgType.PING and self._autoping: + await self.pong(msg.data) + continue + elif msg.type is WSMsgType.PONG and self._autoping: + continue + + return msg + + async def receive_str(self, *, timeout: Optional[float] = None) -> str: + msg = await self.receive(timeout) + if msg.type is not WSMsgType.TEXT: + raise WSMessageTypeError( + f"Received message {msg.type}:{msg.data!r} is not WSMsgType.TEXT" + ) + return cast(str, msg.data) + + async def receive_bytes(self, *, timeout: Optional[float] = None) -> bytes: + msg = await self.receive(timeout) + if msg.type is not WSMsgType.BINARY: + raise WSMessageTypeError( + f"Received message {msg.type}:{msg.data!r} is not WSMsgType.BINARY" + ) + return cast(bytes, msg.data) + + async def receive_json( + self, *, loads: JSONDecoder = json.loads, timeout: Optional[float] = None + ) -> Any: + data = await self.receive_str(timeout=timeout) + return loads(data) + + async def write(self, data: bytes) -> None: + raise RuntimeError("Cannot call .write() for websocket") + + def __aiter__(self) -> "WebSocketResponse": + return self + + async def __anext__(self) -> WSMessage: + msg = await self.receive() + if msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.CLOSED): + raise StopAsyncIteration + return msg + + def _cancel(self, exc: BaseException) -> None: + # web_protocol calls this from connection_lost + # or when the server is shutting down. + self._closing = True + self._cancel_heartbeat() + if self._reader is not None: + set_exception(self._reader, exc) diff --git a/lib/python3.1/site-packages/aiohttp/worker.py b/lib/python3.1/site-packages/aiohttp/worker.py new file mode 100644 index 0000000000000000000000000000000000000000..f7281bfde7541412c3174aa5fdcb859fa1b7a996 --- /dev/null +++ b/lib/python3.1/site-packages/aiohttp/worker.py @@ -0,0 +1,255 @@ +"""Async gunicorn worker for aiohttp.web""" + +import asyncio +import inspect +import os +import re +import signal +import sys +from types import FrameType +from typing import TYPE_CHECKING, Any, Optional + +from gunicorn.config import AccessLogFormat as GunicornAccessLogFormat +from gunicorn.workers import base + +from aiohttp import web + +from .helpers import set_result +from .web_app import Application +from .web_log import AccessLogger + +if TYPE_CHECKING: + import ssl + + SSLContext = ssl.SSLContext +else: + try: + import ssl + + SSLContext = ssl.SSLContext + except ImportError: # pragma: no cover + ssl = None # type: ignore[assignment] + SSLContext = object # type: ignore[misc,assignment] + + +__all__ = ("GunicornWebWorker", "GunicornUVLoopWebWorker") + + +class GunicornWebWorker(base.Worker): # type: ignore[misc,no-any-unimported] + + DEFAULT_AIOHTTP_LOG_FORMAT = AccessLogger.LOG_FORMAT + DEFAULT_GUNICORN_LOG_FORMAT = GunicornAccessLogFormat.default + + def __init__(self, *args: Any, **kw: Any) -> None: # pragma: no cover + super().__init__(*args, **kw) + + self._task: Optional[asyncio.Task[None]] = None + self.exit_code = 0 + self._notify_waiter: Optional[asyncio.Future[bool]] = None + + def init_process(self) -> None: + # create new event_loop after fork + asyncio.get_event_loop().close() + + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + super().init_process() + + def run(self) -> None: + self._task = self.loop.create_task(self._run()) + + try: # ignore all finalization problems + self.loop.run_until_complete(self._task) + except Exception: + self.log.exception("Exception in gunicorn worker") + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) + self.loop.close() + + sys.exit(self.exit_code) + + async def _run(self) -> None: + runner = None + if isinstance(self.wsgi, Application): + app = self.wsgi + elif inspect.iscoroutinefunction(self.wsgi) or ( + sys.version_info < (3, 14) and asyncio.iscoroutinefunction(self.wsgi) + ): + wsgi = await self.wsgi() + if isinstance(wsgi, web.AppRunner): + runner = wsgi + app = runner.app + else: + app = wsgi + else: + raise RuntimeError( + "wsgi app should be either Application or " + "async function returning Application, got {}".format(self.wsgi) + ) + + if runner is None: + access_log = self.log.access_log if self.cfg.accesslog else None + runner = web.AppRunner( + app, + logger=self.log, + keepalive_timeout=self.cfg.keepalive, + access_log=access_log, + access_log_format=self._get_valid_log_format( + self.cfg.access_log_format + ), + shutdown_timeout=self.cfg.graceful_timeout / 100 * 95, + ) + await runner.setup() + + ctx = self._create_ssl_context(self.cfg) if self.cfg.is_ssl else None + + runner = runner + assert runner is not None + server = runner.server + assert server is not None + for sock in self.sockets: + site = web.SockSite( + runner, + sock, + ssl_context=ctx, + ) + await site.start() + + # If our parent changed then we shut down. + pid = os.getpid() + try: + while self.alive: # type: ignore[has-type] + self.notify() + + cnt = server.requests_count + if self.max_requests and cnt > self.max_requests: + self.alive = False + self.log.info("Max requests, shutting down: %s", self) + + elif pid == os.getpid() and self.ppid != os.getppid(): + self.alive = False + self.log.info("Parent changed, shutting down: %s", self) + else: + await self._wait_next_notify() + except BaseException: + pass + + await runner.cleanup() + + def _wait_next_notify(self) -> "asyncio.Future[bool]": + self._notify_waiter_done() + + loop = self.loop + assert loop is not None + self._notify_waiter = waiter = loop.create_future() + self.loop.call_later(1.0, self._notify_waiter_done, waiter) + + return waiter + + def _notify_waiter_done( + self, waiter: Optional["asyncio.Future[bool]"] = None + ) -> None: + if waiter is None: + waiter = self._notify_waiter + if waiter is not None: + set_result(waiter, True) + + if waiter is self._notify_waiter: + self._notify_waiter = None + + def init_signals(self) -> None: + # Set up signals through the event loop API. + + self.loop.add_signal_handler( + signal.SIGQUIT, self.handle_quit, signal.SIGQUIT, None + ) + + self.loop.add_signal_handler( + signal.SIGTERM, self.handle_exit, signal.SIGTERM, None + ) + + self.loop.add_signal_handler( + signal.SIGINT, self.handle_quit, signal.SIGINT, None + ) + + self.loop.add_signal_handler( + signal.SIGWINCH, self.handle_winch, signal.SIGWINCH, None + ) + + self.loop.add_signal_handler( + signal.SIGUSR1, self.handle_usr1, signal.SIGUSR1, None + ) + + self.loop.add_signal_handler( + signal.SIGABRT, self.handle_abort, signal.SIGABRT, None + ) + + # Don't let SIGTERM and SIGUSR1 disturb active requests + # by interrupting system calls + signal.siginterrupt(signal.SIGTERM, False) + signal.siginterrupt(signal.SIGUSR1, False) + # Reset signals so Gunicorn doesn't swallow subprocess return codes + # See: https://github.com/aio-libs/aiohttp/issues/6130 + + def handle_quit(self, sig: int, frame: Optional[FrameType]) -> None: + self.alive = False + + # worker_int callback + self.cfg.worker_int(self) + + # wakeup closing process + self._notify_waiter_done() + + def handle_abort(self, sig: int, frame: Optional[FrameType]) -> None: + self.alive = False + self.exit_code = 1 + self.cfg.worker_abort(self) + sys.exit(1) + + @staticmethod + def _create_ssl_context(cfg: Any) -> "SSLContext": + """Creates SSLContext instance for usage in asyncio.create_server. + + See ssl.SSLSocket.__init__ for more details. + """ + if ssl is None: # pragma: no cover + raise RuntimeError("SSL is not supported.") + + ctx = ssl.SSLContext(cfg.ssl_version) + ctx.load_cert_chain(cfg.certfile, cfg.keyfile) + ctx.verify_mode = cfg.cert_reqs + if cfg.ca_certs: + ctx.load_verify_locations(cfg.ca_certs) + if cfg.ciphers: + ctx.set_ciphers(cfg.ciphers) + return ctx + + def _get_valid_log_format(self, source_format: str) -> str: + if source_format == self.DEFAULT_GUNICORN_LOG_FORMAT: + return self.DEFAULT_AIOHTTP_LOG_FORMAT + elif re.search(r"%\([^\)]+\)", source_format): + raise ValueError( + "Gunicorn's style options in form of `%(name)s` are not " + "supported for the log formatting. Please use aiohttp's " + "format specification to configure access log formatting: " + "http://docs.aiohttp.org/en/stable/logging.html" + "#format-specification" + ) + else: + return source_format + + +class GunicornUVLoopWebWorker(GunicornWebWorker): + def init_process(self) -> None: + import uvloop + + # Close any existing event loop before setting a + # new policy. + asyncio.get_event_loop().close() + + # Setup uvloop policy, so that every + # asyncio.get_event_loop() will create an instance + # of uvloop event loop. + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + + super().init_process() diff --git a/lib/python3.1/site-packages/async_timeout/__init__.py b/lib/python3.1/site-packages/async_timeout/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fe4aa581635bb30ee4e880366b1bbab6227fb148 --- /dev/null +++ b/lib/python3.1/site-packages/async_timeout/__init__.py @@ -0,0 +1,276 @@ +import asyncio +import enum +import sys +from types import TracebackType +from typing import Optional, Type, final + + +__version__ = "5.0.1" + + +__all__ = ("timeout", "timeout_at", "Timeout") + + +def timeout(delay: Optional[float]) -> "Timeout": + """timeout context manager. + + Useful in cases when you want to apply timeout logic around block + of code or in cases when asyncio.wait_for is not suitable. For example: + + >>> async with timeout(0.001): + ... async with aiohttp.get('https://github.com') as r: + ... await r.text() + + + delay - value in seconds or None to disable timeout logic + """ + loop = asyncio.get_running_loop() + if delay is not None: + deadline = loop.time() + delay # type: Optional[float] + else: + deadline = None + return Timeout(deadline, loop) + + +def timeout_at(deadline: Optional[float]) -> "Timeout": + """Schedule the timeout at absolute time. + + deadline argument points on the time in the same clock system + as loop.time(). + + Please note: it is not POSIX time but a time with + undefined starting base, e.g. the time of the system power on. + + >>> async with timeout_at(loop.time() + 10): + ... async with aiohttp.get('https://github.com') as r: + ... await r.text() + + + """ + loop = asyncio.get_running_loop() + return Timeout(deadline, loop) + + +class _State(enum.Enum): + INIT = "INIT" + ENTER = "ENTER" + TIMEOUT = "TIMEOUT" + EXIT = "EXIT" + + +if sys.version_info >= (3, 11): + + class _Expired: + __slots__ = ("_val",) + + def __init__(self, val: bool) -> None: + self._val = val + + def __call__(self) -> bool: + return self._val + + def __bool__(self) -> bool: + return self._val + + def __repr__(self) -> str: + return repr(self._val) + + def __str__(self) -> str: + return str(self._val) + + @final + class Timeout(asyncio.Timeout): # type: ignore[misc] + # Supports full asyncio.Timeout API. + # Also provides several asyncio_timeout specific methods + # for backward compatibility. + def __init__( + self, deadline: Optional[float], loop: asyncio.AbstractEventLoop + ) -> None: + super().__init__(deadline) + + @property + def expired(self) -> _Expired: + # a hacky property hat can provide both roles: + # timeout.expired() from asyncio + # timeout.expired from asyncio_timeout + return _Expired(super().expired()) + + @property + def deadline(self) -> Optional[float]: + return self.when() + + def reject(self) -> None: + """Reject scheduled timeout if any.""" + # cancel is maybe better name but + # task.cancel() raises CancelledError in asyncio world. + self.reschedule(None) + + def shift(self, delay: float) -> None: + """Advance timeout on delay seconds. + + The delay can be negative. + + Raise RuntimeError if shift is called when deadline is not scheduled + """ + deadline = self.when() + if deadline is None: + raise RuntimeError("cannot shift timeout if deadline is not scheduled") + self.reschedule(deadline + delay) + + def update(self, deadline: float) -> None: + """Set deadline to absolute value. + + deadline argument points on the time in the same clock system + as loop.time(). + + If new deadline is in the past the timeout is raised immediately. + + Please note: it is not POSIX time but a time with + undefined starting base, e.g. the time of the system power on. + """ + self.reschedule(deadline) + +else: + + @final + class Timeout: + # Internal class, please don't instantiate it directly + # Use timeout() and timeout_at() public factories instead. + # + # Implementation note: `async with timeout()` is preferred + # over `with timeout()`. + # While technically the Timeout class implementation + # doesn't need to be async at all, + # the `async with` statement explicitly points that + # the context manager should be used from async function context. + # + # This design allows to avoid many silly misusages. + # + # TimeoutError is raised immediately when scheduled + # if the deadline is passed. + # The purpose is to time out as soon as possible + # without waiting for the next await expression. + + __slots__ = ("_deadline", "_loop", "_state", "_timeout_handler", "_task") + + def __init__( + self, deadline: Optional[float], loop: asyncio.AbstractEventLoop + ) -> None: + self._loop = loop + self._state = _State.INIT + + self._task: Optional["asyncio.Task[object]"] = None + self._timeout_handler = None # type: Optional[asyncio.Handle] + if deadline is None: + self._deadline = None # type: Optional[float] + else: + self.update(deadline) + + async def __aenter__(self) -> "Timeout": + self._do_enter() + return self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> Optional[bool]: + self._do_exit(exc_type) + return None + + @property + def expired(self) -> bool: + """Is timeout expired during execution?""" + return self._state == _State.TIMEOUT + + @property + def deadline(self) -> Optional[float]: + return self._deadline + + def reject(self) -> None: + """Reject scheduled timeout if any.""" + # cancel is maybe better name but + # task.cancel() raises CancelledError in asyncio world. + if self._state not in (_State.INIT, _State.ENTER): + raise RuntimeError(f"invalid state {self._state.value}") + self._reject() + + def _reject(self) -> None: + self._task = None + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + def shift(self, delay: float) -> None: + """Advance timeout on delay seconds. + + The delay can be negative. + + Raise RuntimeError if shift is called when deadline is not scheduled + """ + deadline = self._deadline + if deadline is None: + raise RuntimeError("cannot shift timeout if deadline is not scheduled") + self.update(deadline + delay) + + def update(self, deadline: float) -> None: + """Set deadline to absolute value. + + deadline argument points on the time in the same clock system + as loop.time(). + + If new deadline is in the past the timeout is raised immediately. + + Please note: it is not POSIX time but a time with + undefined starting base, e.g. the time of the system power on. + """ + if self._state == _State.EXIT: + raise RuntimeError("cannot reschedule after exit from context manager") + if self._state == _State.TIMEOUT: + raise RuntimeError("cannot reschedule expired timeout") + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._deadline = deadline + if self._state != _State.INIT: + self._reschedule() + + def _reschedule(self) -> None: + assert self._state == _State.ENTER + deadline = self._deadline + if deadline is None: + return + + now = self._loop.time() + if self._timeout_handler is not None: + self._timeout_handler.cancel() + + self._task = asyncio.current_task() + if deadline <= now: + self._timeout_handler = self._loop.call_soon(self._on_timeout) + else: + self._timeout_handler = self._loop.call_at(deadline, self._on_timeout) + + def _do_enter(self) -> None: + if self._state != _State.INIT: + raise RuntimeError(f"invalid state {self._state.value}") + self._state = _State.ENTER + self._reschedule() + + def _do_exit(self, exc_type: Optional[Type[BaseException]]) -> None: + if exc_type is asyncio.CancelledError and self._state == _State.TIMEOUT: + assert self._task is not None + self._timeout_handler = None + self._task = None + raise asyncio.TimeoutError + # timeout has not expired + self._state = _State.EXIT + self._reject() + return None + + def _on_timeout(self) -> None: + assert self._task is not None + self._task.cancel() + self._state = _State.TIMEOUT + # drop the reference early + self._timeout_handler = None diff --git a/lib/python3.1/site-packages/async_timeout/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/site-packages/async_timeout/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a0a1e640293679a05ac9134fca66782c89b6b7a Binary files /dev/null and b/lib/python3.1/site-packages/async_timeout/__pycache__/__init__.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/async_timeout/py.typed b/lib/python3.1/site-packages/async_timeout/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..3b94f915737aba1f12a0f067fdba3726bfe02df5 --- /dev/null +++ b/lib/python3.1/site-packages/async_timeout/py.typed @@ -0,0 +1 @@ +Placeholder diff --git a/lib/python3.1/site-packages/brotli.py b/lib/python3.1/site-packages/brotli.py new file mode 100644 index 0000000000000000000000000000000000000000..9be4ed4bef4490d2a3b4a88045b5ccffe23fb8b6 --- /dev/null +++ b/lib/python3.1/site-packages/brotli.py @@ -0,0 +1,55 @@ +# Copyright 2016 The Brotli Authors. All rights reserved. +# +# Distributed under MIT license. +# See file LICENSE for detail or copy at https://opensource.org/licenses/MIT + +"""Functions to compress and decompress data using the Brotli library.""" + +import _brotli + +# The library version. +version = __version__ = _brotli.__version__ + +# The compression mode. +MODE_GENERIC = _brotli.MODE_GENERIC +MODE_TEXT = _brotli.MODE_TEXT +MODE_FONT = _brotli.MODE_FONT + +# The Compressor object. +Compressor = _brotli.Compressor + +# The Decompressor object. +Decompressor = _brotli.Decompressor + +# Compress a byte string. +def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0): + """Compress a byte string. + + Args: + string (bytes): The input data. + mode (int, optional): The compression mode can be MODE_GENERIC (default), + MODE_TEXT (for UTF-8 format text input) or MODE_FONT (for WOFF 2.0). + quality (int, optional): Controls the compression-speed vs compression- + density tradeoff. The higher the quality, the slower the compression. + Range is 0 to 11. Defaults to 11. + lgwin (int, optional): Base 2 logarithm of the sliding window size. Range + is 10 to 24. Defaults to 22. + lgblock (int, optional): Base 2 logarithm of the maximum input block size. + Range is 16 to 24. If set to 0, the value will be set based on the + quality. Defaults to 0. + + Returns: + The compressed byte string. + + Raises: + brotli.error: If arguments are invalid, or compressor fails. + """ + compressor = Compressor(mode=mode, quality=quality, lgwin=lgwin, + lgblock=lgblock) + return compressor.process(string) + compressor.finish() + +# Decompress a compressed byte string. +decompress = _brotli.decompress + +# Raised if compression or decompression fails. +error = _brotli.error diff --git a/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/INSTALLER b/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/LICENSE b/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..93e41fb6965a930641ba181a3ae4e1008246d9d5 --- /dev/null +++ b/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2021-2025, ContourPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/METADATA b/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..4f04341e81058ea488cfb8ba68a62a4269889a47 --- /dev/null +++ b/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/METADATA @@ -0,0 +1,94 @@ +Metadata-Version: 2.1 +Name: contourpy +Version: 1.3.2 +Summary: Python library for calculating contours of 2D quadrilateral grids +Author-Email: Ian Thomas +License: BSD 3-Clause License + + Copyright (c) 2021-2025, ContourPy Developers. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Science/Research +Classifier: License :: OSI Approved :: BSD License +Classifier: Programming Language :: C++ +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Topic :: Scientific/Engineering :: Information Analysis +Classifier: Topic :: Scientific/Engineering :: Mathematics +Classifier: Topic :: Scientific/Engineering :: Visualization +Project-URL: Homepage, https://github.com/contourpy/contourpy +Project-URL: Changelog, https://contourpy.readthedocs.io/en/latest/changelog.html +Project-URL: Documentation, https://contourpy.readthedocs.io +Project-URL: Repository, https://github.com/contourpy/contourpy +Requires-Python: >=3.10 +Requires-Dist: numpy>=1.23 +Provides-Extra: docs +Requires-Dist: furo; extra == "docs" +Requires-Dist: sphinx>=7.2; extra == "docs" +Requires-Dist: sphinx-copybutton; extra == "docs" +Provides-Extra: bokeh +Requires-Dist: bokeh; extra == "bokeh" +Requires-Dist: selenium; extra == "bokeh" +Provides-Extra: mypy +Requires-Dist: contourpy[bokeh,docs]; extra == "mypy" +Requires-Dist: bokeh; extra == "mypy" +Requires-Dist: docutils-stubs; extra == "mypy" +Requires-Dist: mypy==1.15.0; extra == "mypy" +Requires-Dist: types-Pillow; extra == "mypy" +Provides-Extra: test +Requires-Dist: contourpy[test-no-images]; extra == "test" +Requires-Dist: matplotlib; extra == "test" +Requires-Dist: Pillow; extra == "test" +Provides-Extra: test-no-images +Requires-Dist: pytest; extra == "test-no-images" +Requires-Dist: pytest-cov; extra == "test-no-images" +Requires-Dist: pytest-rerunfailures; extra == "test-no-images" +Requires-Dist: pytest-xdist; extra == "test-no-images" +Requires-Dist: wurlitzer; extra == "test-no-images" +Description-Content-Type: text/markdown + +ContourPy + +ContourPy is a Python library for calculating contours of 2D quadrilateral grids. It is written in C++11 and wrapped using pybind11. + +It contains the 2005 and 2014 algorithms used in Matplotlib as well as a newer algorithm that includes more features and is available in both serial and multithreaded versions. It provides an easy way for Python libraries to use contouring algorithms without having to include Matplotlib as a dependency. + + * **Documentation**: https://contourpy.readthedocs.io + * **Source code**: https://github.com/contourpy/contourpy + +| | | +| --- | --- | +| Latest release | [![PyPI version](https://img.shields.io/pypi/v/contourpy.svg?label=pypi&color=fdae61)](https://pypi.python.org/pypi/contourpy) [![conda-forge version](https://img.shields.io/conda/v/conda-forge/contourpy.svg?label=conda-forge&color=a6d96a)](https://anaconda.org/conda-forge/contourpy) | +| Downloads | [![PyPi downloads](https://img.shields.io/pypi/dm/contourpy?label=pypi&style=flat&color=fdae61)](https://pepy.tech/project/contourpy) | +| Python version | [![Platforms](https://img.shields.io/pypi/pyversions/contourpy?color=fdae61)](https://pypi.org/project/contourpy/) | +| Coverage | [![Codecov](https://img.shields.io/codecov/c/gh/contourpy/contourpy?color=fdae61&label=codecov)](https://app.codecov.io/gh/contourpy/contourpy) | diff --git a/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/RECORD b/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..e9854b618efc367bf57dd8e8dc75834816ac009c --- /dev/null +++ b/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/RECORD @@ -0,0 +1,42 @@ +contourpy-1.3.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +contourpy-1.3.2.dist-info/LICENSE,sha256=NBcJefxk9PXm36Zu8n3sMU___FhSAAxg9INuwd-_FW4,1534 +contourpy-1.3.2.dist-info/METADATA,sha256=DYYPfLivjoUAXCCUmeDKJdqFFbS2eRXui4i4hSwxbAg,5461 +contourpy-1.3.2.dist-info/RECORD,, +contourpy-1.3.2.dist-info/WHEEL,sha256=sZM_NeUMz2G4fDenMf11eikcCxcLaQWiYRmjwQBavQs,137 +contourpy/__init__.py,sha256=Vi2YbtUhM9VxYPY3PBvxfu0xZYr6fBysl5gQPJEo88k,11831 +contourpy/__pycache__/__init__.cpython-310.pyc,, +contourpy/__pycache__/_version.cpython-310.pyc,, +contourpy/__pycache__/array.cpython-310.pyc,, +contourpy/__pycache__/chunk.cpython-310.pyc,, +contourpy/__pycache__/convert.cpython-310.pyc,, +contourpy/__pycache__/dechunk.cpython-310.pyc,, +contourpy/__pycache__/enum_util.cpython-310.pyc,, +contourpy/__pycache__/typecheck.cpython-310.pyc,, +contourpy/__pycache__/types.cpython-310.pyc,, +contourpy/_contourpy.cpython-310-x86_64-linux-gnu.so,sha256=8it_MHwflBYc2jRua5KVBXY14gZO7MuH6GAuQVm8V-E,854312 +contourpy/_contourpy.pyi,sha256=fvtccxkiZwGb6qYag7Fp4E8bsFmAIjAmobf8LNxqfgc,7122 +contourpy/_version.py,sha256=HgKA3RqZvC7slo8MgLyffCGwJbQ3cY6I7oUMFvGLWps,22 +contourpy/array.py,sha256=4WwLuiZe30rizn_raymmY13OzE6hlCsDOO8kuVFOP18,8979 +contourpy/chunk.py,sha256=8njDQqlpuD22RjaaCyA75FXQsSQDY5hZGJSrxFpvGGU,3279 +contourpy/convert.py,sha256=mhyn7prEoWCnf0igaH-VqDwlk-CegFsZ4qOy2LL-hpU,26154 +contourpy/dechunk.py,sha256=EgFL6hw5H54ccuof4tJ2ehdnktT7trgZjiZqppsH8QI,7756 +contourpy/enum_util.py,sha256=o8MItJRs08oqzwPP3IwC75BBAY9Qq95saIzjkXBXwqA,1519 +contourpy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +contourpy/typecheck.py,sha256=t1nvvCuKMYva1Zx4fc30EpdKFcO0Enz3n_UFfXBsq9o,10747 +contourpy/types.py,sha256=2K4T5tJpMIjYrkkg1Lqh3C2ZKlnOhnMtYmtwz92l_y8,247 +contourpy/util/__init__.py,sha256=eVhJ_crOHL7nkG4Kb0dOo7NL4WHMy_Px665aAN_3d-8,118 +contourpy/util/__pycache__/__init__.cpython-310.pyc,, +contourpy/util/__pycache__/_build_config.cpython-310.pyc,, +contourpy/util/__pycache__/bokeh_renderer.cpython-310.pyc,, +contourpy/util/__pycache__/bokeh_util.cpython-310.pyc,, +contourpy/util/__pycache__/data.cpython-310.pyc,, +contourpy/util/__pycache__/mpl_renderer.cpython-310.pyc,, +contourpy/util/__pycache__/mpl_util.cpython-310.pyc,, +contourpy/util/__pycache__/renderer.cpython-310.pyc,, +contourpy/util/_build_config.py,sha256=uFmEfpQ4-HVkN0zZi7c2060J1i2uoaPsvkS4pf0XdiY,1848 +contourpy/util/bokeh_renderer.py,sha256=sOAZKyN11296muuLQ4uhQYCM4eaMwjnxUFrbbIxucmY,13959 +contourpy/util/bokeh_util.py,sha256=wc-S3ewBUYWyIkEv9jkhFySIergjLQl4Z0UEVnE0HhA,2804 +contourpy/util/data.py,sha256=-7SSGMLX_gN-1H2JzpNSEB_EcEF_uMtYdOo_ePRIcg8,2586 +contourpy/util/mpl_renderer.py,sha256=qPWMxDL_n-4dg9gOIHX-Va2UQz6n98eJdIRszpYr2AU,20125 +contourpy/util/mpl_util.py,sha256=0Jz5f-aA9XMWlpO2pDnHbkVgxIiw4SY_ysxf_gACWEo,3452 +contourpy/util/renderer.py,sha256=8CBHzPmVsFPfqsWxqrxGBhqFpJhVeFHFeDzVXAgT8Fc,5118 diff --git a/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/WHEEL b/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..4e4c38ae320920b8f083b87f408214cdecd350d2 --- /dev/null +++ b/lib/python3.1/site-packages/contourpy-1.3.2.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: meson +Root-Is-Purelib: false +Tag: cp310-cp310-manylinux_2_17_x86_64 +Tag: cp310-cp310-manylinux2014_x86_64 + diff --git a/lib/python3.1/site-packages/decord-0.6.0.dist-info/INSTALLER b/lib/python3.1/site-packages/decord-0.6.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.1/site-packages/decord-0.6.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.1/site-packages/decord-0.6.0.dist-info/METADATA b/lib/python3.1/site-packages/decord-0.6.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..1aef6316c0206e61bd893bceaae252582fb06dc4 --- /dev/null +++ b/lib/python3.1/site-packages/decord-0.6.0.dist-info/METADATA @@ -0,0 +1,17 @@ +Metadata-Version: 2.1 +Name: decord +Version: 0.6.0 +Summary: Decord Video Loader +Home-page: https://github.com/dmlc/decord +Maintainer: Decord committers +Maintainer-email: cheungchih@gmail.com +License: APACHE +Platform: UNKNOWN +Classifier: Development Status :: 3 - Alpha +Classifier: Programming Language :: Python :: 3 +Classifier: License :: OSI Approved :: Apache Software License +Requires-Dist: numpy (>=1.14.0) + +UNKNOWN + + diff --git a/lib/python3.1/site-packages/decord-0.6.0.dist-info/RECORD b/lib/python3.1/site-packages/decord-0.6.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..c213cb2f3222016e3aa2a67d06c49d432ab24c78 --- /dev/null +++ b/lib/python3.1/site-packages/decord-0.6.0.dist-info/RECORD @@ -0,0 +1,96 @@ +decord-0.6.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +decord-0.6.0.dist-info/METADATA,sha256=xxbfHI0xijGKytw1rRFgdw4P8XdqGaS34ptvwQzNGaY,422 +decord-0.6.0.dist-info/RECORD,, +decord-0.6.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +decord-0.6.0.dist-info/WHEEL,sha256=0k8JcxMWZX7TJIi6JFgSy7UEc0KxSHdqNxJk5plm2FY,112 +decord-0.6.0.dist-info/top_level.txt,sha256=8TBMC8W9caRfSBphoy47j2wFImKqCOgWKD3JVELo5e0,17 +decord.libs/libXau-b2e5323c.so.6.0.0,sha256=fB-FvDKeEJc5xqYymKjycVd87sn1kq1WIkpWzyXC06o,12904 +decord.libs/libavcodec-bc50294c.so.58.35.100,sha256=BvmzSkk7axWDOINYuB15PX943JEFwmgu8N1TspBHtPU,13286240 +decord.libs/libavdevice-bf61e037.so.58.5.100,sha256=BuUOWuGd728cc_-jCDYe1OZmX1zVMFaJKHSEhUCiAXw,114320 +decord.libs/libavfilter-1e2243e2.so.7.40.101,sha256=ebN6UmtQ1uvNIlWYMZgnZxjCnAlC0f3pYwbkEwQeAcs,3075448 +decord.libs/libavformat-8b46ea57.so.58.20.100,sha256=mh7uj4IwSOkFnTUW1pxXuKEGBPe3Ir-vIyfQIbRmJRg,2399400 +decord.libs/libavutil-2b26904a.so.56.22.100,sha256=8lf7tbJR8LhYC11lKWpNORrii5ik7XFZYdsN8qqjx7k,413208 +decord.libs/libbz2-13e8c345.so.1.0.4,sha256=nahojg9Xor6BTcIq-b47GPFeAryvLip8D2t-Z0sKjGQ,70944 +decord.libs/libpostproc-88b722f8.so.55.3.100,sha256=lPiywLJefszj5mSMIPwlg5wAALB2FteX0CmD3HXv6fg,120064 +decord.libs/libswresample-230d9f46.so.3.3.100,sha256=RFl3l5IcorH3SVU5fMBOjxiiYlxwpfDKAW5fkkxCtZo,124008 +decord.libs/libswscale-8e37dcfd.so.5.3.100,sha256=zEgZOVgnbPGufBJyVUcQqWJxMx-NC08xBhdSyq6MWxM,537808 +decord.libs/libvpx-5769b0ce.so.6.4.0,sha256=ZKiSpxqfTPwEPIdbCymfJ3GeyLi4I_eonBHSbqGK7pE,3717040 +decord.libs/libx264-555fb44a.so.164,sha256=s6-DVzCOtnuxbyzSHSKyg_-1lVn9HMfTiuqmPoTrFdU,2448192 +decord.libs/libxcb-77222338.so.1.1.0,sha256=JmolAiXy5-YsOyFJYJD_i9-cj_dVXH_vgrLgY4XTbas,194008 +decord.libs/libxcb-shape-893f3868.so.0.0.0,sha256=iZkl8PqeV88T2C9IrzZEIurFyncRhkbFpTxukEwFkYM,21712 +decord.libs/libxcb-shm-7ffb2544.so.0.0.0,sha256=nKJAHj-CKN_Pfaf2Hax326OpcnfzoCMeKx5yqYMW8E0,21320 +decord.libs/libxcb-xfixes-6523fc53.so.0.0.0,sha256=fWHDt5GxEsXuQlvhicunNt5CkS41_hwP5hSuF_jAOnk,53472 +decord.libs/libz-eb09ad1d.so.1.2.3,sha256=Occ38Rdxpnqlv4lgkj2inrjERnLkx0ig9Kb0Ce5GMQ4,92080 +decord/__init__.py,sha256=CSDTRqZNkqrV5DymfO4S01jwrhvwfqdLpBGaSN73v7A,573 +decord/__pycache__/__init__.cpython-310.pyc,, +decord/__pycache__/_api_internal.cpython-310.pyc,, +decord/__pycache__/audio_reader.cpython-310.pyc,, +decord/__pycache__/av_reader.cpython-310.pyc,, +decord/__pycache__/base.cpython-310.pyc,, +decord/__pycache__/logging.cpython-310.pyc,, +decord/__pycache__/ndarray.cpython-310.pyc,, +decord/__pycache__/video_loader.cpython-310.pyc,, +decord/__pycache__/video_reader.cpython-310.pyc,, +decord/_api_internal.py,sha256=Tyg95icCBxd718nz5rYx4mRB0_sj2PsL4B5GRfD_Bt4,35 +decord/_ffi/__init__.py,sha256=iL6be84mN7ugukvO__rYZ6QYpATtkgV4NvABp5vJ-bE,350 +decord/_ffi/__pycache__/__init__.cpython-310.pyc,, +decord/_ffi/__pycache__/base.cpython-310.pyc,, +decord/_ffi/__pycache__/function.cpython-310.pyc,, +decord/_ffi/__pycache__/libinfo.cpython-310.pyc,, +decord/_ffi/__pycache__/ndarray.cpython-310.pyc,, +decord/_ffi/__pycache__/runtime_ctypes.cpython-310.pyc,, +decord/_ffi/_ctypes/__init__.py,sha256=MaHmc-jEIpozRSG9f8NDZ85apbg69a1Zx2jfYSKA3iE,44 +decord/_ffi/_ctypes/__pycache__/__init__.cpython-310.pyc,, +decord/_ffi/_ctypes/__pycache__/function.cpython-310.pyc,, +decord/_ffi/_ctypes/__pycache__/ndarray.cpython-310.pyc,, +decord/_ffi/_ctypes/__pycache__/types.cpython-310.pyc,, +decord/_ffi/_ctypes/function.py,sha256=8z-j1bP7gMKn-wQAl2w2CJddWxYUATWdNCq4qozMzOM,8483 +decord/_ffi/_ctypes/ndarray.py,sha256=G5JT_HkznwamIAsJeWezpgoUE_NsN_3St5_8ssjW9wg,3850 +decord/_ffi/_ctypes/types.py,sha256=XYFBGbUZCRBNlXLF6I0yt1nrUekEIa5_39QsWLqU-rE,2075 +decord/_ffi/_cy2/__init__.py,sha256=bRydCazovWKmvE9pMp-ePEAOrowmCimHmwi_06d3kg0,24 +decord/_ffi/_cy2/__pycache__/__init__.cpython-310.pyc,, +decord/_ffi/_cy3/__init__.py,sha256=8BeIXQdCkzXQHHqf5pc6ckzNO5WTM7F59u8DNPHZYCI,24 +decord/_ffi/_cy3/__pycache__/__init__.cpython-310.pyc,, +decord/_ffi/base.py,sha256=sDdrYqiOZf6XMSV6Onbv7GALImJsuyYPWvA0DTvi550,3345 +decord/_ffi/function.py,sha256=sX2ZQbw98vO7nCMAdiwgWOlpX1E5cF9xswvwbscuU-Y,9170 +decord/_ffi/libinfo.py,sha256=k7jSh5TdBfmok5Dt7H1He9WXzfeuRTgyogrpBNtSgAA,3260 +decord/_ffi/ndarray.py,sha256=qq0EebywVM_AYq7CdphuJXwcPEUXkyo0kSI6eUn5-PU,10500 +decord/_ffi/runtime_ctypes.py,sha256=1TFGEhFVHjlRQIjonXOkI8LdbWX1zb4i2KPzVMnGwuw,7311 +decord/audio_reader.py,sha256=AeWdPmdO92n1OUfCvQI3DuTA8ZJ7ChOlDfyDkImhrB0,6455 +decord/av_reader.py,sha256=tl3BNsvWA0xM3yKsRDkKlIlvArfU7ERVOlU3toZQf70,7042 +decord/base.py,sha256=S_2c-vum29MWFXyiE0kpKCxgINGzW9v1PbmOhHETiFk,540 +decord/bridge/__init__.py,sha256=APGQ8LQbOAgC07rvV5K-lCfTclZvIuwGzP4ZKqx1ks0,2360 +decord/bridge/__pycache__/__init__.cpython-310.pyc,, +decord/bridge/__pycache__/mxnet.cpython-310.pyc,, +decord/bridge/__pycache__/tf.cpython-310.pyc,, +decord/bridge/__pycache__/torchdl.cpython-310.pyc,, +decord/bridge/__pycache__/tvm.cpython-310.pyc,, +decord/bridge/__pycache__/utils.cpython-310.pyc,, +decord/bridge/mxnet.py,sha256=r-SGX0WcR1cETiHHPozQXIVyhcgREJtImtyFNcRocIM,732 +decord/bridge/tf.py,sha256=O_ZAvgrkGw9_CnBu_HLxl2foINM_MHtHn___JcoLIqY,964 +decord/bridge/torchdl.py,sha256=zK44YA2ZAM3ggtIgr8n0rmlbuj4wRfwQakCt_rpLoi0,1041 +decord/bridge/tvm.py,sha256=vw5dZ42fgNCoNWSEKCEZThLL9y1fSjYPN4hlX5XX0zA,699 +decord/bridge/utils.py,sha256=ep_Pfj9pbK8ZtEEgzm49ZKreGI2eNtEhG-aGfIbpe7U,623 +decord/data/__init__.py,sha256=ByuAOSddqCxE8O2v1t9dJ1XC4rw4arABk_UJw2EMKyE,37 +decord/data/__pycache__/__init__.cpython-310.pyc,, +decord/data/__pycache__/base_action.cpython-310.pyc,, +decord/data/__pycache__/dataloader.cpython-310.pyc,, +decord/data/base_action.py,sha256=vhuj5tV25vdAlLDSCt2qa6GYoZJQONHeOpYJ3PykCHE,15897 +decord/data/dataloader.py,sha256=z6TkVRJaPlvLjdAFSggPAMbOVarMjv-qaPFB3kmNhi0,4899 +decord/data/kinetics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +decord/data/kinetics/__pycache__/__init__.cpython-310.pyc,, +decord/data/kinetics/__pycache__/kinetics400_action.cpython-310.pyc,, +decord/data/kinetics/kinetics400_action.py,sha256=giEJHvr-IkUFNNSCwNM0GABnYsxgJeDcwjxdjJrVn0E,13787 +decord/data/transforms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +decord/data/transforms/__pycache__/__init__.cpython-310.pyc,, +decord/data/transforms/__pycache__/action.cpython-310.pyc,, +decord/data/transforms/action.py,sha256=saA_oA8nzeJXBoADJxqaGPjWmGnnenn_7KbQO4xQ37M,17123 +decord/function/__init__.py,sha256=7fE_Lh3ohn-_NcdGJZqYGDN6oQXltCOv2pXFxd4zlWk,126 +decord/function/__pycache__/__init__.cpython-310.pyc,, +decord/function/__pycache__/base.cpython-310.pyc,, +decord/function/base.py,sha256=zx61H929vivx4Yx5JIZ6R4GnnmdWAVm_iPajbBSr_Ds,998 +decord/libdecord.so,sha256=mLJgxYEhBmSLopknmRb76YQ5iT40bU79z1zeZrqJc9o,12465984 +decord/logging.py,sha256=ksBvL5S2OwET752N2bopYECcpVvrQGx7YF0mDq91Vek,339 +decord/ndarray.py,sha256=ijnG2qqx_5mN5mNcjRK4OcABAeniiiBx15znF0Uf26g,1984 +decord/video_loader.py,sha256=SFzoUaYAlY_8IWpGmO6uEKL0QhOCRPR_3_-ihx5GpcE,3889 +decord/video_reader.py,sha256=2HF1GmWcNKu4_j7X_7-O58G7xCQSekV1_F8APdvASFc,8604 diff --git a/lib/python3.1/site-packages/decord-0.6.0.dist-info/REQUESTED b/lib/python3.1/site-packages/decord-0.6.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.1/site-packages/decord-0.6.0.dist-info/WHEEL b/lib/python3.1/site-packages/decord-0.6.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..aa767d9ffb7fbd844b4c36ca9803c53ec2c7ad14 --- /dev/null +++ b/lib/python3.1/site-packages/decord-0.6.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.36.2) +Root-Is-Purelib: false +Tag: cp36-cp36m-manylinux2010_x86_64 + diff --git a/lib/python3.1/site-packages/decord-0.6.0.dist-info/top_level.txt b/lib/python3.1/site-packages/decord-0.6.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..308106f0c9142ae48ac55cdf3bf085aac57b9371 --- /dev/null +++ b/lib/python3.1/site-packages/decord-0.6.0.dist-info/top_level.txt @@ -0,0 +1 @@ +decord diff --git a/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/INSTALLER b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/METADATA b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..2beb20a76b1bb5f263caed76e566577c7122676e --- /dev/null +++ b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/METADATA @@ -0,0 +1,447 @@ +Metadata-Version: 2.4 +Name: deepspeed +Version: 0.17.4 +Summary: DeepSpeed library +Home-page: http://deepspeed.ai +Author: DeepSpeed Team +Author-email: deepspeed-info@microsoft.com +License: Apache Software License 2.0 +Project-URL: Documentation, https://deepspeed.readthedocs.io +Project-URL: Source, https://github.com/deepspeedai/DeepSpeed +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Description-Content-Type: text/markdown +License-File: LICENSE +Requires-Dist: einops +Requires-Dist: hjson +Requires-Dist: msgpack +Requires-Dist: ninja +Requires-Dist: numpy +Requires-Dist: packaging>=20.0 +Requires-Dist: psutil +Requires-Dist: py-cpuinfo +Requires-Dist: pydantic>=2.0.0 +Requires-Dist: torch +Requires-Dist: tqdm +Requires-Dist: nvidia-ml-py +Provides-Extra: 1bit +Requires-Dist: cupy-cuda12x; extra == "1bit" +Provides-Extra: 1bit-mpi +Requires-Dist: mpi4py; extra == "1bit-mpi" +Requires-Dist: cupy-cuda12x; extra == "1bit-mpi" +Provides-Extra: readthedocs +Requires-Dist: autodoc_pydantic>=2.0.0; extra == "readthedocs" +Requires-Dist: docutils<0.18; extra == "readthedocs" +Requires-Dist: hjson; extra == "readthedocs" +Requires-Dist: packaging; extra == "readthedocs" +Requires-Dist: psutil; extra == "readthedocs" +Requires-Dist: py-cpuinfo; extra == "readthedocs" +Requires-Dist: pydantic>=2.0.0; extra == "readthedocs" +Requires-Dist: recommonmark; extra == "readthedocs" +Requires-Dist: sphinx_rtd_theme; extra == "readthedocs" +Requires-Dist: torch; extra == "readthedocs" +Requires-Dist: tqdm; extra == "readthedocs" +Provides-Extra: dev +Requires-Dist: accelerate; extra == "dev" +Requires-Dist: clang-format==18.1.3; extra == "dev" +Requires-Dist: comet_ml>=3.41.0; extra == "dev" +Requires-Dist: deepspeed-kernels; sys_platform == "linux" and extra == "dev" +Requires-Dist: docutils<0.18; extra == "dev" +Requires-Dist: future; extra == "dev" +Requires-Dist: importlib-metadata>=4; extra == "dev" +Requires-Dist: mup; extra == "dev" +Requires-Dist: pre-commit>=3.2.0; extra == "dev" +Requires-Dist: pytest<8.4.0,>=7.2.0; extra == "dev" +Requires-Dist: pytest-forked; extra == "dev" +Requires-Dist: pytest-randomly; extra == "dev" +Requires-Dist: pytest-xdist; extra == "dev" +Requires-Dist: qtorch==0.3.0; extra == "dev" +Requires-Dist: recommonmark; extra == "dev" +Requires-Dist: sphinx; extra == "dev" +Requires-Dist: sphinx-rtd-theme; extra == "dev" +Requires-Dist: tensorboard; extra == "dev" +Requires-Dist: torchvision; extra == "dev" +Requires-Dist: transformers>=4.51.3; extra == "dev" +Requires-Dist: wandb; extra == "dev" +Provides-Extra: autotuning +Requires-Dist: tabulate; extra == "autotuning" +Provides-Extra: autotuning-ml +Requires-Dist: hjson; extra == "autotuning-ml" +Requires-Dist: tabulate; extra == "autotuning-ml" +Requires-Dist: xgboost; extra == "autotuning-ml" +Provides-Extra: sparse-attn +Requires-Dist: triton==1.0.0; extra == "sparse-attn" +Provides-Extra: sparse +Requires-Dist: neural-compressor==2.1.0; extra == "sparse" +Provides-Extra: inf +Requires-Dist: google; extra == "inf" +Requires-Dist: lm-eval==0.3.0; extra == "inf" +Requires-Dist: protobuf; extra == "inf" +Requires-Dist: qtorch; extra == "inf" +Requires-Dist: safetensors; extra == "inf" +Requires-Dist: sentencepiece; extra == "inf" +Requires-Dist: transformers>=4.32.1; extra == "inf" +Provides-Extra: sd +Requires-Dist: diffusers>=0.25.0; extra == "sd" +Requires-Dist: triton>=2.1.0; extra == "sd" +Provides-Extra: triton +Requires-Dist: triton==2.1.0; extra == "triton" +Provides-Extra: deepcompile +Requires-Dist: scipy; extra == "deepcompile" +Provides-Extra: all +Requires-Dist: safetensors; extra == "all" +Requires-Dist: pydantic>=2.0.0; extra == "all" +Requires-Dist: neural-compressor==2.1.0; extra == "all" +Requires-Dist: torchvision; extra == "all" +Requires-Dist: py-cpuinfo; extra == "all" +Requires-Dist: accelerate; extra == "all" +Requires-Dist: diffusers>=0.25.0; extra == "all" +Requires-Dist: wandb; extra == "all" +Requires-Dist: hjson; extra == "all" +Requires-Dist: qtorch==0.3.0; extra == "all" +Requires-Dist: comet_ml>=3.41.0; extra == "all" +Requires-Dist: tqdm; extra == "all" +Requires-Dist: pytest-xdist; extra == "all" +Requires-Dist: pre-commit>=3.2.0; extra == "all" +Requires-Dist: docutils<0.18; extra == "all" +Requires-Dist: transformers>=4.51.3; extra == "all" +Requires-Dist: triton==2.1.0; extra == "all" +Requires-Dist: protobuf; extra == "all" +Requires-Dist: mpi4py; extra == "all" +Requires-Dist: sphinx-rtd-theme; extra == "all" +Requires-Dist: sentencepiece; extra == "all" +Requires-Dist: torch; extra == "all" +Requires-Dist: xgboost; extra == "all" +Requires-Dist: future; extra == "all" +Requires-Dist: recommonmark; extra == "all" +Requires-Dist: lm-eval==0.3.0; extra == "all" +Requires-Dist: psutil; extra == "all" +Requires-Dist: clang-format==18.1.3; extra == "all" +Requires-Dist: cupy-cuda12x; extra == "all" +Requires-Dist: triton>=2.1.0; extra == "all" +Requires-Dist: sphinx_rtd_theme; extra == "all" +Requires-Dist: packaging; extra == "all" +Requires-Dist: google; extra == "all" +Requires-Dist: pytest<8.4.0,>=7.2.0; extra == "all" +Requires-Dist: pytest-forked; extra == "all" +Requires-Dist: deepspeed-kernels; sys_platform == "linux" and extra == "all" +Requires-Dist: scipy; extra == "all" +Requires-Dist: qtorch; extra == "all" +Requires-Dist: mup; extra == "all" +Requires-Dist: importlib-metadata>=4; extra == "all" +Requires-Dist: autodoc_pydantic>=2.0.0; extra == "all" +Requires-Dist: tensorboard; extra == "all" +Requires-Dist: tabulate; extra == "all" +Requires-Dist: sphinx; extra == "all" +Requires-Dist: transformers>=4.32.1; extra == "all" +Requires-Dist: triton==1.0.0; extra == "all" +Requires-Dist: pytest-randomly; extra == "all" +Dynamic: author +Dynamic: author-email +Dynamic: classifier +Dynamic: description +Dynamic: description-content-type +Dynamic: home-page +Dynamic: license +Dynamic: license-file +Dynamic: project-url +Dynamic: provides-extra +Dynamic: requires-dist +Dynamic: summary + +[![License Apache 2.0](https://badgen.net/badge/license/apache2.0/blue)](https://github.com/deepspeedai/DeepSpeed/blob/master/LICENSE) +[![PyPI version](https://badge.fury.io/py/deepspeed.svg)](https://pypi.org/project/deepspeed/) +[![Downloads](https://static.pepy.tech/badge/deepspeed)](https://pepy.tech/project/deepspeed) +[![Build](https://badgen.net/badge/build/check-status/blue)](#build-pipeline-status) +[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/9530/badge)](https://www.bestpractices.dev/projects/9530) +[![Twitter](https://img.shields.io/twitter/follow/DeepSpeedAI)](https://twitter.com/intent/follow?screen_name=DeepSpeedAI) +[![Japanese Twitter](https://img.shields.io/badge/%E6%97%A5%E6%9C%AC%E8%AA%9ETwitter-%40DeepSpeedAI_JP-blue)](https://twitter.com/DeepSpeedAI_JP) +[![Chinese Zhihu](https://img.shields.io/badge/%E7%9F%A5%E4%B9%8E-%E5%BE%AE%E8%BD%AFDeepSpeed-blue)](https://www.zhihu.com/people/deepspeed) + + +
    + + +
    + +## Latest News + DeepSpeed empowers ChatGPT-like model training with a single click, offering 15x speedup over SOTA RLHF systems with unprecedented cost reduction at all scales; [learn how](https://github.com/deepspeedai/DeepSpeed/tree/master/blogs/deepspeed-chat). +* [2025/06] [Arctic Long Sequence Training (ALST) with DeepSpeed: Scalable And Efficient Training For Multi-Million Token Sequences](https://www.snowflake.com/en/engineering-blog/arctic-long-sequence-training-multi-million-token-ai/) +* [2025/04] [DeepCompile: Unlocking Compiler Optimization for Distributed Training](https://github.com/deepspeedai/DeepSpeed/blob/master/blogs/deepcompile/README.md) +* [2025/03] [DeepSpeed-AutoTP: Automatic Tensor Parallel Training of Hugging Face models](https://github.com/deepspeedai/DeepSpeed/blob/master/blogs/huggingface-tp/README.md) +* [2024/12] [Ulysses-Offload: Democratizing Long Context LLM Training ](https://github.com/deepspeedai/DeepSpeed/blob/master/blogs/ulysses-offload/README.md) +* [2024/12] [DeepSpeed-Domino: Communication-Free LLM Training Engine](https://github.com/deepspeedai/DeepSpeed/blob/master/blogs/deepspeed-domino/README.md) +* [2024/08] [DeepSpeed on Windows](https://github.com/deepspeedai/DeepSpeed/tree/master/blogs/windows/08-2024/README.md) [[日本語](https://github.com/deepspeedai/DeepSpeed/tree/master/blogs/windows/08-2024/japanese/README.md)] [[中文](https://github.com/deepspeedai/DeepSpeed/tree/master/blogs/windows/08-2024/chinese/README.md)] + + +
    + More news + +
    + +--- + +# Extreme Speed and Scale for DL Training and Inference + +***[DeepSpeed](https://www.deepspeed.ai/) enabled the world's most powerful language models (at the time of this writing) such as [MT-530B](https://www.microsoft.com/en-us/research/blog/using-deepspeed-and-megatron-to-train-megatron-turing-nlg-530b-the-worlds-largest-and-most-powerful-generative-language-model/) and [BLOOM](https://huggingface.co/blog/bloom-megatron-deepspeed)***. It is an easy-to-use deep learning optimization software suite that powers unprecedented scale and speed for both training and inference. With DeepSpeed you can: + +* Train/Inference dense or sparse models with billions or trillions of parameters +* Achieve excellent system throughput and efficiently scale to thousands of GPUs +* Train/Inference on resource constrained GPU systems +* Achieve unprecedented low latency and high throughput for inference +* Achieve extreme compression for an unparalleled inference latency and model size reduction with low costs + +--- + +# DeepSpeed's four innovation pillars + + + + +## DeepSpeed-Training + +DeepSpeed offers a confluence of system innovations, that has made large scale DL training effective, and efficient, greatly improved ease of use, and redefined the DL training landscape in terms of scale that is possible. These innovations such as ZeRO, 3D-Parallelism, DeepSpeed-MoE, ZeRO-Infinity, etc. fall under the training pillar. Learn more: [DeepSpeed-Training](https://www.deepspeed.ai/training/) + +## DeepSpeed-Inference + +DeepSpeed brings together innovations in parallelism technology such as tensor, pipeline, expert and ZeRO-parallelism, and combines them with high performance custom inference kernels, communication optimizations and heterogeneous memory technologies to enable inference at an unprecedented scale, while achieving unparalleled latency, throughput and cost reduction. This systematic composition of system technologies for inference falls under the inference pillar. Learn more: [DeepSpeed-Inference](https://www.deepspeed.ai/inference) + + +## DeepSpeed-Compression + +To further increase the inference efficiency, DeepSpeed offers easy-to-use and flexible-to-compose compression techniques for researchers and practitioners to compress their models while delivering faster speed, smaller model size, and significantly reduced compression cost. Moreover, SoTA innovations on compression like ZeroQuant and XTC are included under the compression pillar. Learn more: [DeepSpeed-Compression](https://www.deepspeed.ai/compression) + +## DeepSpeed4Science + +In line with Microsoft's mission to solve humanity's most pressing challenges, the DeepSpeed team at Microsoft is responding to this opportunity by launching a new initiative called *DeepSpeed4Science*, aiming to build unique capabilities through AI system technology innovations to help domain experts to unlock today's biggest science mysteries. Learn more: [tutorials](https://www.deepspeed.ai/deepspeed4science/) + +--- + +# DeepSpeed Software Suite + +## DeepSpeed Library + + The [DeepSpeed](https://github.com/deepspeedai/deepspeed) library (this repository) implements and packages the innovations and technologies in DeepSpeed Training, Inference and Compression Pillars into a single easy-to-use, open-sourced repository. It allows for easy composition of multitude of features within a single training, inference or compression pipeline. The DeepSpeed Library is heavily adopted by the DL community, and has been used to enable some of the most powerful models (see [DeepSpeed Adoption](#deepspeed-adoption)). + +## Model Implementations for Inference (MII) + + [Model Implementations for Inference (MII)](https://github.com/deepspeedai/deepspeed-mii) is an open-sourced repository for making low-latency and high-throughput inference accessible to all data scientists by alleviating the need to apply complex system optimization techniques themselves. Out-of-box, MII offers support for thousands of widely used DL models, optimized using DeepSpeed-Inference, that can be deployed with a few lines of code, while achieving significant latency reduction compared to their vanilla open-sourced versions. + +## DeepSpeed on Azure + + DeepSpeed users are diverse and have access to different environments. We recommend to try DeepSpeed on Azure as it is the simplest and easiest method. The recommended method to try DeepSpeed on Azure is through AzureML [recipes](https://github.com/Azure/azureml-examples/tree/main/v1/python-sdk/workflows/train/deepspeed). The job submission and data preparation scripts have been made available [here](https://github.com/deepspeedai/Megatron-DeepSpeed/tree/main/examples_deepspeed/azureml). For more details on how to use DeepSpeed on Azure, please follow the [Azure tutorial](https://www.deepspeed.ai/tutorials/azure/). + +--- + +# DeepSpeed Adoption + +DeepSpeed was an important part of Microsoft’s +[AI at Scale](https://www.microsoft.com/en-us/research/project/ai-at-scale/) +initiative to enable next-generation AI capabilities at scale, where you can find more +information [here](https://innovation.microsoft.com/en-us/exploring-ai-at-scale). + +DeepSpeed has been used to train many different large-scale models, below is a list of several examples that we are aware of (if you'd like to include your model please submit a PR): + + * [Megatron-Turing NLG (530B)](https://www.microsoft.com/en-us/research/blog/using-deepspeed-and-megatron-to-train-megatron-turing-nlg-530b-the-worlds-largest-and-most-powerful-generative-language-model/) + * [Jurassic-1 (178B)](https://uploads-ssl.webflow.com/60fd4503684b466578c0d307/61138924626a6981ee09caf6_jurassic_tech_paper.pdf) + * [BLOOM (176B)](https://huggingface.co/blog/bloom-megatron-deepspeed) + * [GLM (130B)](https://github.com/THUDM/GLM-130B) + * [xTrimoPGLM (100B)](https://www.biorxiv.org/content/10.1101/2023.07.05.547496v2) + * [YaLM (100B)](https://github.com/yandex/YaLM-100B) + * [GPT-NeoX (20B)](https://github.com/EleutherAI/gpt-neox) + * [AlexaTM (20B)](https://www.amazon.science/blog/20b-parameter-alexa-model-sets-new-marks-in-few-shot-learning) + * [Turing NLG (17B)](https://www.microsoft.com/en-us/research/blog/turing-nlg-a-17-billion-parameter-language-model-by-microsoft/) + * [METRO-LM (5.4B)](https://arxiv.org/pdf/2204.06644.pdf) + +DeepSpeed has been integrated with several different popular open-source DL frameworks such as: + +| | Documentation | +| ---------------------------------------------------------------------------------------------- | -------------------------------------------- | + | [Transformers with DeepSpeed](https://huggingface.co/docs/transformers/deepspeed) | +| | [Accelerate with DeepSpeed](https://huggingface.co/docs/accelerate/usage_guides/deepspeed) | +| | [Lightning with DeepSpeed](https://lightning.ai/docs/pytorch/stable/advanced/model_parallel.html#deepspeed) | +| | [MosaicML with DeepSpeed](https://docs.mosaicml.com/projects/composer/en/latest/trainer/using_the_trainer.html?highlight=deepspeed#deepspeed-integration) | +| | [Determined with DeepSpeed](https://docs.determined.ai/latest/training/apis-howto/deepspeed/overview.html) | +| | [MMEngine with DeepSpeed](https://mmengine.readthedocs.io/en/latest/common_usage/large_model_training.html#deepspeed) | + +--- + +# Build Pipeline Status + +| Description | Status | +| ----------- | ------ | +| NVIDIA | [![nv-torch-latest-v100](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-torch-latest-v100.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-torch-latest-v100.yml) [![nv-inference](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-inference.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-inference.yml) [![nv-nightly](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-nightly.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-nightly.yml) | +| AMD | [![amd-mi200](https://github.com/deepspeedai/DeepSpeed/actions/workflows/amd-mi200.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/amd-mi200.yml) | +| CPU | [![torch-latest-cpu](https://github.com/deepspeedai/DeepSpeed/actions/workflows/cpu-torch-latest.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/cpu-torch-latest.yml) | +| Intel Gaudi | [![hpu-gaudi2](https://github.com/deepspeedai/DeepSpeed/actions/workflows/hpu-gaudi2.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/hpu-gaudi2.yml) | +| Intel XPU | [![xpu-max1100](https://github.com/deepspeedai/DeepSpeed/actions/workflows/xpu-max1100.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/xpu-max1100.yml) | +| PyTorch Nightly | [![nv-torch-nightly-v100](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-torch-nightly-v100.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-torch-nightly-v100.yml) | +| Integrations | [![nv-transformers-v100](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-transformers-v100.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-transformers-v100.yml) [![nv-lightning-v100](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-lightning-v100.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-lightning-v100.yml) [![nv-accelerate-v100](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-accelerate-v100.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-accelerate-v100.yml) [![nv-mii](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-mii.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-mii.yml) [![nv-ds-chat](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-ds-chat.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-ds-chat.yml) [![nv-sd](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-sd.yml/badge.svg)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/nv-sd.yml) | +| Misc | [![Formatting](https://github.com/deepspeedai/DeepSpeed/actions/workflows/formatting.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/formatting.yml) [![pages-build-deployment](https://github.com/deepspeedai/DeepSpeed/actions/workflows/pages/pages-build-deployment/badge.svg)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/pages/pages-build-deployment) [![Documentation Status](https://readthedocs.org/projects/deepspeed/badge/?version=latest)](https://deepspeed.readthedocs.io/en/latest/?badge=latest)[![python](https://github.com/deepspeedai/DeepSpeed/actions/workflows/python.yml/badge.svg?branch=master)](https://github.com/deepspeedai/DeepSpeed/actions/workflows/python.yml) | +| Huawei Ascend NPU | [![Huawei Ascend NPU](https://github.com/Ascend/Ascend-CI/actions/workflows/deepspeed.yaml/badge.svg?branch=main)](https://github.com/Ascend/Ascend-CI/actions/workflows/deepspeed.yaml) | + +# Installation + +The quickest way to get started with DeepSpeed is via pip, this will install +the latest release of DeepSpeed which is not tied to specific PyTorch or CUDA +versions. DeepSpeed includes several C++/CUDA extensions that we commonly refer +to as our 'ops'. By default, all of these extensions/ops will be built +just-in-time (JIT) using [torch's JIT C++ extension loader that relies on +ninja](https://pytorch.org/docs/stable/cpp_extension.html) to build and +dynamically link them at runtime. + +## Requirements +* [PyTorch](https://pytorch.org/) must be installed _before_ installing DeepSpeed. +* For full feature support we recommend a version of PyTorch that is >= 1.9 and ideally the latest PyTorch stable release. +* A CUDA or ROCm compiler such as [nvcc](https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/#introduction) or [hipcc](https://github.com/ROCm-Developer-Tools/HIPCC) used to compile C++/CUDA/HIP extensions. +* Specific GPUs we develop and test against are listed below, this doesn't mean your GPU will not work if it doesn't fall into this category it's just DeepSpeed is most well tested on the following: + * NVIDIA: Pascal, Volta, Ampere, and Hopper architectures + * AMD: MI100 and MI200 + +## Contributed HW support +* DeepSpeed now support various HW accelerators. + +| Contributor | Hardware | Accelerator Name | Contributor validated | Upstream validated | +|-------------|-------------------------------------|------------------| --------------------- |--------------------| +| Huawei | Huawei Ascend NPU | npu | Yes | No | +| Intel | Intel(R) Gaudi(R) 2 AI accelerator | hpu | Yes | Yes | +| Intel | Intel(R) Xeon(R) Processors | cpu | Yes | Yes | +| Intel | Intel(R) Data Center GPU Max series | xpu | Yes | Yes | +| Tecorigin | Scalable Data Analytics Accelerator | sdaa | Yes | No | + +## PyPI +We regularly push releases to [PyPI](https://pypi.org/project/deepspeed/) and encourage users to install from there in most cases. + +```bash +pip install deepspeed +``` + +After installation, you can validate your install and see which extensions/ops +your machine is compatible with via the DeepSpeed environment report. + +```bash +ds_report +``` + +If you would like to pre-install any of the DeepSpeed extensions/ops (instead +of JIT compiling) or install pre-compiled ops via PyPI please see our [advanced +installation instructions](https://www.deepspeed.ai/tutorials/advanced-install/). + +## Windows +Many DeepSpeed features are supported on Windows for both training and inference. You can read more about this in the original blog post [here](https://github.com/deepspeedai/DeepSpeed/tree/master/blogs/windows/08-2024/README.md). Among features that are currently not supported are async io (AIO) and GDS (which does not support Windows). +1. Install PyTorch, such as pytorch 2.3+cu121. +2. Install Visual C++ build tools, such as VS2022 C++ x64/x86 build tools. +3. Launch Cmd console with Administrator permissions for creating required symlink folders and ensure MSVC tools are added to your PATH or launch the Developer Command Prompt for Visual Studio 2022 with administrator permissions. +4. Run `build_win.bat` to build wheel in `dist` folder. + +# Features + +Please checkout [DeepSpeed-Training](https://www.deepspeed.ai/training), [DeepSpeed-Inference](https://www.deepspeed.ai/inference) and [DeepSpeed-Compression](https://www.deepspeed.ai/compression) pages for full set of features offered along each of these three pillars. + +# Further Reading + +All DeepSpeed documentation, tutorials, and blogs can be found on our website: [deepspeed.ai](https://www.deepspeed.ai/) + + +| | Description | +| ---------------------------------------------------------------------------------------------- | -------------------------------------------- | +| [Getting Started](https://www.deepspeed.ai/getting-started/) | First steps with DeepSpeed | +| [DeepSpeed JSON Configuration](https://www.deepspeed.ai/docs/config-json/) | Configuring DeepSpeed | +| [API Documentation](https://deepspeed.readthedocs.io/en/latest/) | Generated DeepSpeed API documentation | +| [Tutorials](https://www.deepspeed.ai/tutorials/) | Tutorials | +| [Blogs](https://www.deepspeed.ai/posts/) | Blogs | + + +# Contributing +DeepSpeed welcomes your contributions! Please see our +[contributing](CONTRIBUTING.md) guide for more details on formatting, testing, +etc.
    +Thanks so much to all of our amazing contributors! + + + + + +## Contributor License Agreement +This project welcomes contributions and suggestions. Most contributions require you to +agree to a Contributor License Agreement (CLA) declaring that you have the right to, and +actually do, grant us the rights to use your contribution. For details, visit +https://cla.opensource.microsoft.com. + +When you submit a pull request, a CLA bot will automatically determine whether you need +to provide a CLA and decorate the PR appropriately (e.g., status check, comment). Simply +follow the instructions provided by the bot. You will only need to do this once across +all repos using our CLA. + +## Code of Conduct +This project has adopted the [Microsoft Open Source Code of +Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the +[Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact +[opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + +# Publications +1. Samyam Rajbhandari, Jeff Rasley, Olatunji Ruwase, Yuxiong He. (2019) ZeRO: memory optimizations toward training trillion parameter models. [arXiv:1910.02054](https://arxiv.org/abs/1910.02054) and [In Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis (SC '20)](https://dl.acm.org/doi/10.5555/3433701.3433727). +2. Jeff Rasley, Samyam Rajbhandari, Olatunji Ruwase, and Yuxiong He. (2020) DeepSpeed: System Optimizations Enable Training Deep Learning Models with Over 100 Billion Parameters. [In Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining (KDD '20, Tutorial)](https://dl.acm.org/doi/10.1145/3394486.3406703). +3. Minjia Zhang, Yuxiong He. (2020) Accelerating Training of Transformer-Based Language Models with Progressive Layer Dropping. [arXiv:2010.13369](https://arxiv.org/abs/2010.13369) and [NeurIPS 2020](https://proceedings.neurips.cc/paper/2020/hash/a1140a3d0df1c81e24ae954d935e8926-Abstract.html). +4. Jie Ren, Samyam Rajbhandari, Reza Yazdani Aminabadi, Olatunji Ruwase, Shuangyan Yang, Minjia Zhang, Dong Li, Yuxiong He. (2021) ZeRO-Offload: Democratizing Billion-Scale Model Training. [arXiv:2101.06840](https://arxiv.org/abs/2101.06840) and [USENIX ATC 2021](https://www.usenix.org/conference/atc21/presentation/ren-jie). [[paper]](https://arxiv.org/abs/2101.06840) [[slides]](https://www.usenix.org/system/files/atc21_slides_ren-jie.pdf) [[blog]](https://www.microsoft.com/en-us/research/blog/deepspeed-extreme-scale-model-training-for-everyone/) +5. Hanlin Tang, Shaoduo Gan, Ammar Ahmad Awan, Samyam Rajbhandari, Conglong Li, Xiangru Lian, Ji Liu, Ce Zhang, Yuxiong He. (2021) 1-bit Adam: Communication Efficient Large-Scale Training with Adam's Convergence Speed. [arXiv:2102.02888](https://arxiv.org/abs/2102.02888) and [ICML 2021](http://proceedings.mlr.press/v139/tang21a.html). +6. Samyam Rajbhandari, Olatunji Ruwase, Jeff Rasley, Shaden Smith, Yuxiong He. (2021) ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning. [arXiv:2104.07857](https://arxiv.org/abs/2104.07857) and [SC 2021](https://dl.acm.org/doi/abs/10.1145/3458817.3476205). [[paper]](https://arxiv.org/abs/2104.07857) [[slides]](docs/assets/files/SC21-ZeRO-Infinity.pdf) [[blog]](https://www.microsoft.com/en-us/research/blog/zero-infinity-and-deepspeed-unlocking-unprecedented-model-scale-for-deep-learning-training/) +7. Conglong Li, Ammar Ahmad Awan, Hanlin Tang, Samyam Rajbhandari, Yuxiong He. (2021) 1-bit LAMB: Communication Efficient Large-Scale Large-Batch Training with LAMB's Convergence Speed. [arXiv:2104.06069](https://arxiv.org/abs/2104.06069) and [HiPC 2022](https://hipc.org/advance-program/). +8. Conglong Li, Minjia Zhang, Yuxiong He. (2021) The Stability-Efficiency Dilemma: Investigating Sequence Length Warmup for Training GPT Models. [arXiv:2108.06084](https://arxiv.org/abs/2108.06084) and [NeurIPS 2022](https://openreview.net/forum?id=JpZ5du_Kdh). +9. Yucheng Lu, Conglong Li, Minjia Zhang, Christopher De Sa, Yuxiong He. (2022) Maximizing Communication Efficiency for Large-scale Training via 0/1 Adam. [arXiv:2202.06009](https://arxiv.org/abs/2202.06009). +10. Samyam Rajbhandari, Conglong Li, Zhewei Yao, Minjia Zhang, Reza Yazdani Aminabadi, Ammar Ahmad Awan, Jeff Rasley, Yuxiong He. (2022) DeepSpeed-MoE: Advancing Mixture-of-Experts Inference and Training to Power Next-Generation AI Scale [arXiv:2201.05596](https://arxiv.org/abs/2201.05596) and [ICML 2022](https://proceedings.mlr.press/v162/rajbhandari22a.html). [[pdf]](https://arxiv.org/abs/2201.05596) [[slides]](docs/assets/files/ICML-5mins.pdf) [[blog]](https://www.microsoft.com/en-us/research/blog/deepspeed-advancing-moe-inference-and-training-to-power-next-generation-ai-scale/) +11. Shaden Smith, Mostofa Patwary, Brandon Norick, Patrick LeGresley, Samyam Rajbhandari, Jared Casper, Zhun Liu, Shrimai Prabhumoye, George Zerveas, Vijay Korthikanti, Elton Zhang, Rewon Child, Reza Yazdani Aminabadi, Julie Bernauer, Xia Song, Mohammad Shoeybi, Yuxiong He, Michael Houston, Saurabh Tiwary, Bryan Catanzaro. (2022) Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model [arXiv:2201.11990](https://arxiv.org/abs/2201.11990). +12. Xiaoxia Wu, Zhewei Yao, Minjia Zhang, Conglong Li, Yuxiong He. (2022) Extreme Compression for Pre-trained Transformers Made Simple and Efficient. [arXiv:2206.01859](https://arxiv.org/abs/2206.01859) and [NeurIPS 2022](https://openreview.net/forum?id=xNeAhc2CNAl). +13. Zhewei Yao, Reza Yazdani Aminabadi, Minjia Zhang, Xiaoxia Wu, Conglong Li, Yuxiong He. (2022) ZeroQuant: Efficient and Affordable Post-Training Quantization for Large-Scale Transformers. [arXiv:2206.01861](https://arxiv.org/abs/2206.01861) and [NeurIPS 2022](https://openreview.net/forum?id=f-fVCElZ-G1) [[slides]](docs/assets/files/zeroquant_series.pdf) [[blog]](https://www.microsoft.com/en-us/research/blog/deepspeed-compression-a-composable-library-for-extreme-compression-and-zero-cost-quantization/) +14. Reza Yazdani Aminabadi, Samyam Rajbhandari, Minjia Zhang, Ammar Ahmad Awan, Cheng Li, Du Li, Elton Zheng, Jeff Rasley, Shaden Smith, Olatunji Ruwase, Yuxiong He. (2022) DeepSpeed Inference: Enabling Efficient Inference of Transformer Models at Unprecedented Scale. [arXiv:2207.00032](https://arxiv.org/abs/2207.00032) and [SC 2022](https://dl.acm.org/doi/abs/10.5555/3571885.3571946). [[paper]](https://arxiv.org/abs/2207.00032) [[slides]](docs/assets/files/sc22-ds-inference.pdf) [[blog]](https://www.microsoft.com/en-us/research/blog/deepspeed-accelerating-large-scale-model-inference-and-training-via-system-optimizations-and-compression/) +15. Zhewei Yao, Xiaoxia Wu, Conglong Li, Connor Holmes, Minjia Zhang, Cheng Li, Yuxiong He. (2022) Random-LTD: Random and Layerwise Token Dropping Brings Efficient Training for Large-scale Transformers. [arXiv:2211.11586](https://arxiv.org/abs/2211.11586). +16. Conglong Li, Zhewei Yao, Xiaoxia Wu, Minjia Zhang, Yuxiong He. (2022) DeepSpeed Data Efficiency: Improving Deep Learning Model Quality and Training Efficiency via Efficient Data Sampling and Routing. [arXiv:2212.03597](https://arxiv.org/abs/2212.03597) [ENLSP2023 Workshop at NeurIPS2023](https://neurips2023-enlsp.github.io/) +17. Xiaoxia Wu, Cheng Li, Reza Yazdani Aminabadi, Zhewei Yao, Yuxiong He. (2023) Understanding INT4 Quantization for Transformer Models: Latency Speedup, Composability, and Failure Cases. [arXiv:2301.12017](https://arxiv.org/abs/2301.12017) and [ICML2023](https://icml.cc/Conferences/2023). +18. Syed Zawad, Cheng Li, Zhewei Yao, Elton Zheng, Yuxiong He, Feng Yan. (2023) DySR: Adaptive Super-Resolution via Algorithm and System Co-design. [ICLR:2023](https://openreview.net/forum?id=Pgtn4l6eKjv). +19. Sheng Shen, Zhewei Yao, Chunyuan Li, Trevor Darrell, Kurt Keutzer, Yuxiong He. (2023) Scaling Vision-Language Models with Sparse Mixture of Experts. [arXiv:2303.07226](https://arxiv.org/abs/2303.07226) and [Finding at EMNLP2023](https://2023.emnlp.org/). +20. Quentin Anthony, Ammar Ahmad Awan, Jeff Rasley, Yuxiong He, Aamir Shafi, Mustafa Abduljabbar, Hari Subramoni, Dhabaleswar Panda. (2023) MCR-DL: Mix-and-Match Communication Runtime for Deep Learning [arXiv:2303.08374](https://arxiv.org/abs/2303.08374) and will appear at IPDPS 2023. +21. Siddharth Singh, Olatunji Ruwase, Ammar Ahmad Awan, Samyam Rajbhandari, Yuxiong He, Abhinav Bhatele. (2023) A Hybrid Tensor-Expert-Data Parallelism Approach to Optimize Mixture-of-Experts Training [arXiv:2303.06318](https://arxiv.org/abs/2303.06318) and [ICS 2023](https://dl.acm.org/doi/10.1145/3577193.3593704). +22. Guanhua Wang, Heyang Qin, Sam Ade Jacobs, Xiaoxia Wu, Connor Holmes, Zhewei Yao, Samyam Rajbhandari, Olatunji Ruwase, Feng Yan, Lei Yang, Yuxiong He. (2023) ZeRO++: Extremely Efficient Collective Communication for Giant Model Training [arXiv:2306.10209](https://arxiv.org/abs/2306.10209) and [ML for Sys Workshop at NeurIPS2023](http://mlforsystems.org/) [[blog]](https://www.microsoft.com/en-us/research/blog/deepspeed-zero-a-leap-in-speed-for-llm-and-chat-model-training-with-4x-less-communication/) +23. Zhewei Yao, Xiaoxia Wu, Cheng Li, Stephen Youn, Yuxiong He. (2023) ZeroQuant-V2: Exploring Post-training Quantization in LLMs from Comprehensive Study to Low Rank Compensation [arXiv:2303.08302](https://arxiv.org/abs/2303.08302) and [ENLSP2023 Workshop at NeurIPS2023](https://neurips2023-enlsp.github.io/) [[slides]](docs/assets/files/zeroquant_series.pdf) +24. Pareesa Ameneh Golnari, Zhewei Yao, Yuxiong He. (2023) Selective Guidance: Are All the Denoising Steps of Guided Diffusion Important? [arXiv:2305.09847](https://arxiv.org/abs/2305.09847) +25. Zhewei Yao, Reza Yazdani Aminabadi, Olatunji Ruwase, Samyam Rajbhandari, Xiaoxia Wu, Ammar Ahmad Awan, Jeff Rasley, Minjia Zhang, Conglong Li, Connor Holmes, Zhongzhu Zhou, Michael Wyatt, Molly Smith, Lev Kurilenko, Heyang Qin, Masahiro Tanaka, Shuai Che, Shuaiwen Leon Song, Yuxiong He. (2023) DeepSpeed-Chat: Easy, Fast and Affordable RLHF Training of ChatGPT-like Models at All Scales [arXiv:2308.01320](https://arxiv.org/abs/2308.01320). +26. Xiaoxia Wu, Zhewei Yao, Yuxiong He. (2023) ZeroQuant-FP: A Leap Forward in LLMs Post-Training W4A8 Quantization Using Floating-Point Formats [arXiv:2307.09782](https://arxiv.org/abs/2307.09782) and [ENLSP2023 Workshop at NeurIPS2023](https://neurips2023-enlsp.github.io/) [[slides]](docs/assets/files/zeroquant_series.pdf) +27. Zhewei Yao, Xiaoxia Wu, Conglong Li, Minjia Zhang, Heyang Qin, Olatunji Ruwase, Ammar Ahmad Awan, Samyam Rajbhandari, Yuxiong He. (2023) DeepSpeed-VisualChat: Multi-Round Multi-Image Interleave Chat via Multi-Modal Causal Attention [arXiv:2309.14327](https://arxiv.org/pdf/2309.14327.pdf) +28. Shuaiwen Leon Song, Bonnie Kruft, Minjia Zhang, Conglong Li, Shiyang Chen, Chengming Zhang, Masahiro Tanaka, Xiaoxia Wu, Jeff Rasley, Ammar Ahmad Awan, Connor Holmes, Martin Cai, Adam Ghanem, Zhongzhu Zhou, Yuxiong He, et al. (2023) DeepSpeed4Science Initiative: Enabling Large-Scale Scientific Discovery through Sophisticated AI System Technologies [arXiv:2310.04610](https://arxiv.org/abs/2310.04610) [[blog]](https://www.microsoft.com/en-us/research/blog/announcing-the-deepspeed4science-initiative-enabling-large-scale-scientific-discovery-through-sophisticated-ai-system-technologies/) +29. Zhewei Yao, Reza Yazdani Aminabadi, Stephen Youn, Xiaoxia Wu, Elton Zheng, Yuxiong He. (2023) ZeroQuant-HERO: Hardware-Enhanced Robust Optimized Post-Training Quantization Framework for W8A8 Transformers [arXiv:2310.17723](https://arxiv.org/abs/2310.17723) + +30. Xiaoxia Wu, Haojun Xia, Stephen Youn, Zhen Zheng, Shiyang Chen, Arash Bakhtiari, Michael Wyatt, Reza Yazdani Aminabadi, Yuxiong He, Olatunji Ruwase, Leon Song, Zhewei Yao (2023) ZeroQuant(4+2): Redefining LLMs Quantization with a New FP6-Centric Strategy for Diverse Generative Tasks [arXiv:2312.08583](https://arxiv.org/abs/2312.08583) + +31. Haojun Xia, Zhen Zheng, Xiaoxia Wu, Shiyang Chen, Zhewei Yao, Stephen Youn, Arash Bakhtiari, Michael Wyatt, Donglin Zhuang, Zhongzhu Zhou, Olatunji Ruwase, Yuxiong He, Shuaiwen Leon Song. (2024) FP6-LLM: Efficiently Serving Large Language Models Through FP6-Centric Algorithm-System Co-Design [arXiv:2401.14112](https://arxiv.org/abs/2401.14112) +32. Sam Ade Jacobs, Masahiro Tanaka, Chengming Zhang, Minjia Zhang, Reza Yazdani Aminadabi, Shuaiwen Leon Song, Samyam Rajbhandari, Yuxiong He. (2024) [System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models](https://dl.acm.org/doi/10.1145/3662158.3662806) +33. Xinyu Lian, Sam Ade Jacobs, Lev Kurilenko, Masahiro Tanaka, Stas Bekman, Olatunji Ruwase, Minjia Zhang. (2024) Universal Checkpointing: Efficient and Flexible Checkpointing for Large Scale Distributed Training [arXiv:2406.18820](https://arxiv.org/abs/2406.18820) +34. Stas Bekman, Samyam Rajbhandari, Michael Wyatt, Jeff Rasley, Tunji Ruwase, Zhewei Yao, Aurick Qiao, Yuxiong He. (2025) Arctic Long Sequence Training: Scalable And Efficient Training For Multi-Million Token Sequences [arXiv:2506.13996](https://arxiv.org/abs/2506.13996) + + +# Videos +1. DeepSpeed KDD 2020 Tutorial + 1. [Overview](https://www.youtube.com/watch?v=CaseqC45DNc&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=29) + 2. [ZeRO + large model training](https://www.youtube.com/watch?v=y4_bCiAsIAk&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=28) + 3. [17B T-NLG demo](https://www.youtube.com/watch?v=9V-ZbP92drg&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=27) + 4. [Fastest BERT training + RScan tuning](https://www.youtube.com/watch?v=o1K-ZG9F6u0&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=26) + 5. DeepSpeed hands on deep dive: [part 1](https://www.youtube.com/watch?v=_NOk-mBwDYg&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=92), [part 2](https://www.youtube.com/watch?v=sG6_c4VXLww&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=94), [part 3](https://www.youtube.com/watch?v=k9yPkBTayos&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=93) + 6. [FAQ](https://www.youtube.com/watch?v=nsHu6vEgPew&list=PLa85ZdUjfWS21mgibJ2vCvLziprjpKoW0&index=24) +2. Microsoft Research Webinar + * Registration is free and all videos are available on-demand. + * [ZeRO & Fastest BERT: Increasing the scale and speed of deep learning training in DeepSpeed](https://note.microsoft.com/MSR-Webinar-DeepSpeed-Registration-On-Demand.html). +3. [DeepSpeed on AzureML](https://youtu.be/yBVXR8G8Bg8) +4. [Large Model Training and Inference with DeepSpeed // Samyam Rajbhandari // LLMs in Prod Conference](https://www.youtube.com/watch?v=cntxC3g22oU) [[slides]](docs/assets/files/presentation-mlops.pdf) +5. Community Tutorials + * [DeepSpeed: All the tricks to scale to gigantic models (Mark Saroufim)](https://www.youtube.com/watch?v=pDGI668pNg0) + * [Turing-NLG, DeepSpeed and the ZeRO optimizer (Yannic Kilcher)](https://www.youtube.com/watch?v=tC01FRB0M7w) + * [Ultimate Guide To Scaling ML Models (The AI Epiphany)](https://www.youtube.com/watch?v=hc0u4avAkuM) diff --git a/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/RECORD b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..32638f601b210c4cb3aa3ca319d9324e020860e5 --- /dev/null +++ b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/RECORD @@ -0,0 +1,1580 @@ +../../../bin/deepspeed,sha256=LxBeXOHYIWx_Du2krffIsSYZvUkxC21OODOHzsdSvdU,163 +../../../bin/deepspeed.pt,sha256=LxBeXOHYIWx_Du2krffIsSYZvUkxC21OODOHzsdSvdU,163 +../../../bin/ds,sha256=LxBeXOHYIWx_Du2krffIsSYZvUkxC21OODOHzsdSvdU,163 +../../../bin/ds_bench,sha256=JIPKWDZbvxt1hq4djCFTD558ureO4tUhoDb1PlLCirk,913 +../../../bin/ds_elastic,sha256=qFh0WQTeGR1-ruYciC9hJ8dDDeP8C7Jk265m-hvXaqs,1926 +../../../bin/ds_io,sha256=MldwDMKE4KgxvShFFjJsSXAHKg27uvfBXZugiWD4uFQ,164 +../../../bin/ds_nvme_tune,sha256=8qDwe8frlJBRtVu-T7BzWY43h08tgrmdXC4t3vmrbfo,342 +../../../bin/ds_report,sha256=nuxo9W5WAs7yVXZmLEY288A8SqKUD1XiET5AXRuovZU,166 +../../../bin/ds_ssh,sha256=BT6cdZ47ceJiKPK40eVf9NCXF5RLjayEwWlyZ7viWbE,680 +../../../bin/dsr,sha256=nuxo9W5WAs7yVXZmLEY288A8SqKUD1XiET5AXRuovZU,166 +deepspeed-0.17.4.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +deepspeed-0.17.4.dist-info/METADATA,sha256=gFgcjs2LqKYICUTlG-0xwpAteKWZh1pVwKvzE3naq_c,42377 +deepspeed-0.17.4.dist-info/RECORD,, +deepspeed-0.17.4.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +deepspeed-0.17.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91 +deepspeed-0.17.4.dist-info/entry_points.txt,sha256=btWVtkuERrrqyxQojUVMJZOeaoNPz0pjiZKaJ_Q4MJ8,84 +deepspeed-0.17.4.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357 +deepspeed-0.17.4.dist-info/top_level.txt,sha256=VK6WJ48PiB2ke4oVIo2mLzqXYqGXdu4MPE2GP-vfvA8,10 +deepspeed/__init__.py,sha256=tcS3eXx2HvMLr38_ofg-2RC9l9k1s3AsYX-TyJEBrJU,16832 +deepspeed/__pycache__/__init__.cpython-310.pyc,, +deepspeed/__pycache__/constants.cpython-310.pyc,, +deepspeed/__pycache__/env_report.cpython-310.pyc,, +deepspeed/__pycache__/git_version_info.cpython-310.pyc,, +deepspeed/__pycache__/git_version_info_installed.cpython-310.pyc,, +deepspeed/accelerator/__init__.py,sha256=1pNnvGX9h418ekjY_Xnpb363tLo-5Szw3icQf9hRvk0,248 +deepspeed/accelerator/__pycache__/__init__.cpython-310.pyc,, +deepspeed/accelerator/__pycache__/abstract_accelerator.cpython-310.pyc,, +deepspeed/accelerator/__pycache__/cpu_accelerator.cpython-310.pyc,, +deepspeed/accelerator/__pycache__/cuda_accelerator.cpython-310.pyc,, +deepspeed/accelerator/__pycache__/hpu_accelerator.cpython-310.pyc,, +deepspeed/accelerator/__pycache__/mlu_accelerator.cpython-310.pyc,, +deepspeed/accelerator/__pycache__/mps_accelerator.cpython-310.pyc,, +deepspeed/accelerator/__pycache__/npu_accelerator.cpython-310.pyc,, +deepspeed/accelerator/__pycache__/real_accelerator.cpython-310.pyc,, +deepspeed/accelerator/__pycache__/sdaa_accelerator.cpython-310.pyc,, +deepspeed/accelerator/__pycache__/xpu_accelerator.cpython-310.pyc,, +deepspeed/accelerator/abstract_accelerator.py,sha256=mF-Km8bgg_yGqwcoPcSheKRlYoyFP4i1cFzxUHRL72o,5840 +deepspeed/accelerator/cpu_accelerator.py,sha256=Mn58Mq3eZJr5QWmQliXOY3-D5c1ne-uzmYgeiz1oexM,10553 +deepspeed/accelerator/cuda_accelerator.py,sha256=Nik1PGGOw-WBQti2seBiFfjeKhYGUcGPEE5pQlfxIE8,12923 +deepspeed/accelerator/hpu_accelerator.py,sha256=U8MLbGI_qP6PnTwP2F7ibzUalB0-MxnhsJr9Ld0F3BA,10971 +deepspeed/accelerator/mlu_accelerator.py,sha256=1tUxv8mnAHOfo4AY4BxiQ4rCD8KULFKZVeNRULOQNB4,9337 +deepspeed/accelerator/mps_accelerator.py,sha256=XkqWL7P86KFp2ebtGSNF6pPsR5OPsFm1Jl5UPEcR6C4,7006 +deepspeed/accelerator/npu_accelerator.py,sha256=FwTKiHZeaWW5PngtqZYFS49w5hKFmQwhsOPR1KreX2s,9020 +deepspeed/accelerator/real_accelerator.py,sha256=CJS2JocHzQCeGJRGQqxpGdzgLJBHqzR31PDB_SaFxN0,12791 +deepspeed/accelerator/sdaa_accelerator.py,sha256=YXdjlEwIUYuocPsu-0_etZH-2ak4yCNQYAFGL-_DKgM,10813 +deepspeed/accelerator/xpu_accelerator.py,sha256=AopBKrqyTu4_GLr1LpTeH075PUQ2iparLBU56ceDng8,10936 +deepspeed/autotuning/__init__.py,sha256=y0O9XDcX76E6dmhNFmRhy9TKfl0ywJrdkXmM2JuzErU,129 +deepspeed/autotuning/__pycache__/__init__.cpython-310.pyc,, +deepspeed/autotuning/__pycache__/autotuner.cpython-310.pyc,, +deepspeed/autotuning/__pycache__/config.cpython-310.pyc,, +deepspeed/autotuning/__pycache__/constants.cpython-310.pyc,, +deepspeed/autotuning/__pycache__/scheduler.cpython-310.pyc,, +deepspeed/autotuning/__pycache__/utils.cpython-310.pyc,, +deepspeed/autotuning/autotuner.py,sha256=_ucGsohYkJ1MFsz1JuB9DJyky-brugx_IDz0mbhqeY0,54335 +deepspeed/autotuning/config.py,sha256=pg5OeryVqrg32xak1NLEWaFiN1MYujdwlSgTuAz3Xl0,4633 +deepspeed/autotuning/config_templates/template_zero0.json,sha256=hR1baTMA5HzfTD1JlXjiTYjMtB9C8CFfxIBzqZeW2Sw,48 +deepspeed/autotuning/config_templates/template_zero1.json,sha256=AAIox-1dviiYg-Z7L-02bxItSnaggAlHkR4TekXXEIw,113 +deepspeed/autotuning/config_templates/template_zero2.json,sha256=FfrPKZHzMJuHn_TVJ7UZ-2s9yJ5IzBIKgT7nLYnIHh4,237 +deepspeed/autotuning/config_templates/template_zero3.json,sha256=WvsoMMp5MDgdVLq1RX1fhKsKbGSHQtpplZzGJikZvzs,485 +deepspeed/autotuning/constants.py,sha256=sr02-8B5ORdimK5VZ1N9FK1luWadmECFLSlMnM-uIxA,5943 +deepspeed/autotuning/scheduler.py,sha256=4IjLO56K-ZXr-HK1FuSuw5-v4tUdAWRZfH8R02WNx3U,15703 +deepspeed/autotuning/tuner/__init__.py,sha256=c9ImdL2iEc89lFltZ2PPe2EOUCuB3Olq3hRP1Q-yUMo,235 +deepspeed/autotuning/tuner/__pycache__/__init__.cpython-310.pyc,, +deepspeed/autotuning/tuner/__pycache__/base_tuner.cpython-310.pyc,, +deepspeed/autotuning/tuner/__pycache__/cost_model.cpython-310.pyc,, +deepspeed/autotuning/tuner/__pycache__/index_based_tuner.cpython-310.pyc,, +deepspeed/autotuning/tuner/__pycache__/model_based_tuner.cpython-310.pyc,, +deepspeed/autotuning/tuner/__pycache__/utils.cpython-310.pyc,, +deepspeed/autotuning/tuner/base_tuner.py,sha256=psA1I4-AEfqGrGLvYYUl3EjM8jG8AlhUXz-AnrFZ7G4,2754 +deepspeed/autotuning/tuner/cost_model.py,sha256=Uu9jD65cvdUK6aozm6Du3XJQZ_W-yOyUrSWFLdOT0do,1820 +deepspeed/autotuning/tuner/index_based_tuner.py,sha256=AEkTByT3XXCyYolAqBiMquV6XHN-ntGmYKMLZKPCRYE,1158 +deepspeed/autotuning/tuner/model_based_tuner.py,sha256=f_CrbgZNONRjdMuRhrJkQbhtih6GOg4LdkXqRDRDzaM,5614 +deepspeed/autotuning/tuner/utils.py,sha256=o5nD51Z6LBylJDRnDNIiOARlBHnOgINBVjBos6ypLEg,2329 +deepspeed/autotuning/utils.py,sha256=d4ePhVxVtekB7gIEK1dIOs6bhxOpmFF8yhKPC_rNRDc,15045 +deepspeed/checkpoint/__init__.py,sha256=3qhX6swvx0-zo72xLl99eZzBr_KC7PguCVCgemYyAGw,576 +deepspeed/checkpoint/__pycache__/__init__.cpython-310.pyc,, +deepspeed/checkpoint/__pycache__/constants.cpython-310.pyc,, +deepspeed/checkpoint/__pycache__/deepspeed_checkpoint.cpython-310.pyc,, +deepspeed/checkpoint/__pycache__/ds_to_universal.cpython-310.pyc,, +deepspeed/checkpoint/__pycache__/reshape_3d_utils.cpython-310.pyc,, +deepspeed/checkpoint/__pycache__/reshape_meg_2d.cpython-310.pyc,, +deepspeed/checkpoint/__pycache__/reshape_utils.cpython-310.pyc,, +deepspeed/checkpoint/__pycache__/universal_checkpoint.cpython-310.pyc,, +deepspeed/checkpoint/__pycache__/utils.cpython-310.pyc,, +deepspeed/checkpoint/__pycache__/zero_checkpoint.cpython-310.pyc,, +deepspeed/checkpoint/constants.py,sha256=FQ55iGvLxiXitjitVfepx3-vLrPx7J-btyfLUGyPGCc,3510 +deepspeed/checkpoint/deepspeed_checkpoint.py,sha256=7Zc6aq_KU6HVt8cnswmP2cA0anOuvdl_nJ9bid3G57w,13231 +deepspeed/checkpoint/ds_to_universal.py,sha256=IbP-9e1A6Y95I6AWLolK1OCHjJ78Ndrp9i07zaLqJN0,22788 +deepspeed/checkpoint/reshape_3d_utils.py,sha256=pmnOtdI3srRS46R9sXLh9kijXPxT3bFJ_c6lWdygbwc,4674 +deepspeed/checkpoint/reshape_meg_2d.py,sha256=o5dpF3CbrbtEt1twz0XsL3xbqCfLfEse7O2ke-WR9_s,7885 +deepspeed/checkpoint/reshape_utils.py,sha256=n3k8xjXn2--nRzcs7jo7V9lFlma7LTLXjkk9S3IgUhM,3458 +deepspeed/checkpoint/universal_checkpoint.py,sha256=EF1VbI2eeHdHXmdgYbHCYVOIf8sjSTcGqY82qjtCtmQ,6788 +deepspeed/checkpoint/utils.py,sha256=Wog7wLoR_F88UOSlLRAX8XspdjA9mF3CiJ5iOYVWh9E,2693 +deepspeed/checkpoint/zero_checkpoint.py,sha256=OsB42DLdUAeuqFbynEHaJGAo3UTbuog1UicXg8dVJAU,5441 +deepspeed/comm/__init__.py,sha256=N31DOMdAdlfSnQd64slK89dpW_y1Vc9oPVZabLArcYw,137 +deepspeed/comm/__pycache__/__init__.cpython-310.pyc,, +deepspeed/comm/__pycache__/backend.cpython-310.pyc,, +deepspeed/comm/__pycache__/ccl.cpython-310.pyc,, +deepspeed/comm/__pycache__/comm.cpython-310.pyc,, +deepspeed/comm/__pycache__/config.cpython-310.pyc,, +deepspeed/comm/__pycache__/constants.cpython-310.pyc,, +deepspeed/comm/__pycache__/reduce_op.cpython-310.pyc,, +deepspeed/comm/__pycache__/torch.cpython-310.pyc,, +deepspeed/comm/__pycache__/utils.cpython-310.pyc,, +deepspeed/comm/backend.py,sha256=TPFMx6ViU4n_NcfwWS56_Yp35PeCi-0Kh9za7UELeyA,1416 +deepspeed/comm/ccl.py,sha256=wGGn3sFUvbzblOsA4uMfKCltk6giyI2dkG6SkmMFzVc,8614 +deepspeed/comm/comm.py,sha256=JyczpTGC1ygq5lKCgfoRcoL_lTyC56Z1dHnp0El6COc,36207 +deepspeed/comm/config.py,sha256=vd3v2i0X638bgqgVENb03GddaJfZVJYV17k3dRHdmKM,728 +deepspeed/comm/constants.py,sha256=Adtnb5LCEjRpylnwgsMAFRmWgPvxb8CEEDvn4-D39lw,1298 +deepspeed/comm/reduce_op.py,sha256=A_KxQtpfUrNun6Rn8SX6iQiuoVgfYM7bTtAYc-YTENE,259 +deepspeed/comm/torch.py,sha256=GzD0zIQ2kzqWXsCZhDX7Sk6Z3bLXqo4Z11264XDYxYE,19980 +deepspeed/comm/utils.py,sha256=I7hJ2Jq5HEPaJDD09eEI9xV5VJZeDdag0ovTeAc3Ex8,3842 +deepspeed/compile/__init__.py,sha256=-DiusbPt_BUtHIDjxVtMiLEuFDUb6ANm671qfLuUXVY,126 +deepspeed/compile/__pycache__/__init__.cpython-310.pyc,, +deepspeed/compile/__pycache__/backend.cpython-310.pyc,, +deepspeed/compile/__pycache__/config.cpython-310.pyc,, +deepspeed/compile/__pycache__/fx.cpython-310.pyc,, +deepspeed/compile/__pycache__/graph_param.cpython-310.pyc,, +deepspeed/compile/__pycache__/inductor.cpython-310.pyc,, +deepspeed/compile/__pycache__/init_z1.cpython-310.pyc,, +deepspeed/compile/__pycache__/init_z3.cpython-310.pyc,, +deepspeed/compile/__pycache__/input_storage.cpython-310.pyc,, +deepspeed/compile/__pycache__/list_schedule.cpython-310.pyc,, +deepspeed/compile/__pycache__/partitioner.cpython-310.pyc,, +deepspeed/compile/__pycache__/patch_compiled_func.cpython-310.pyc,, +deepspeed/compile/__pycache__/patch_fake_tensor.cpython-310.pyc,, +deepspeed/compile/__pycache__/util.cpython-310.pyc,, +deepspeed/compile/backend.py,sha256=7J3IuJJwMKDuH5hg5Ot4nwJhjDbesQxGRwT9-SV3pKc,15885 +deepspeed/compile/config.py,sha256=n-0B63IyyOlK6j-bfUSfVlCxdvJHMDdjPmadKYrb2MQ,1521 +deepspeed/compile/fx.py,sha256=WY5Shuq80GsEj1r3vNd20azPZNfcbMbGkr-c6JfMLVY,4717 +deepspeed/compile/graph_param.py,sha256=KBsOiXs6KfQ_p3UAakfOoTFQfvKXaEc-zNcnRh33V3w,3802 +deepspeed/compile/inductor.py,sha256=dfQC6RSYFZbR2_IXtfkrhNl88t31QZjosp159k477q0,8933 +deepspeed/compile/init_z1.py,sha256=slw23gEhnglbkN1UciCkrgClPmY1cCIf7JHWABwf-n0,3488 +deepspeed/compile/init_z3.py,sha256=59J3fEVt0J1cS_hAjE13tj59SO4RUyE2xv72khpVJOg,3905 +deepspeed/compile/input_storage.py,sha256=u71Xm3YBa5TV7in8SvqzF7PJA4iVO0Fjyw9YlRibwfc,7835 +deepspeed/compile/list_schedule.py,sha256=S2d9OolXfOwpyUkE68QHslWUPOmGWCBc8ijx2qCAsjc,15870 +deepspeed/compile/partitioner.py,sha256=fOtJtoRgoULNvYImDDgAnYli0OH3khT3W0GI98dM2Q8,6902 +deepspeed/compile/passes/__init__.py,sha256=YPYE70D8HDgY-Wo0z8HgOF7ikgUN0F29deid37kdF8k,1470 +deepspeed/compile/passes/__pycache__/__init__.cpython-310.pyc,, +deepspeed/compile/passes/__pycache__/offload_activation.cpython-310.pyc,, +deepspeed/compile/passes/__pycache__/offload_adam_states.cpython-310.pyc,, +deepspeed/compile/passes/__pycache__/offload_parameters.cpython-310.pyc,, +deepspeed/compile/passes/__pycache__/prefetch.cpython-310.pyc,, +deepspeed/compile/passes/__pycache__/selective_gather.cpython-310.pyc,, +deepspeed/compile/passes/__pycache__/zero1_compile.cpython-310.pyc,, +deepspeed/compile/passes/__pycache__/zero3_compile.cpython-310.pyc,, +deepspeed/compile/passes/offload_activation.py,sha256=fPCEG4OtfyMdLEtVCzZYrlvkeZkAeiVX29qh7tlrt_0,3849 +deepspeed/compile/passes/offload_adam_states.py,sha256=vprSD81ZODjObxQFOpkAoawQoyD1nL_X2DVYeKX8YpY,20605 +deepspeed/compile/passes/offload_parameters.py,sha256=zOOIpmbra727IUfSb-Rihuco-PVcTF2cJMfys626ZbM,2046 +deepspeed/compile/passes/prefetch.py,sha256=6NW_QLnIw6VzV5edhmb6sonp0Wu-TSqRPh7KhMO4Ho0,7696 +deepspeed/compile/passes/selective_gather.py,sha256=icQWdK73H61VaD54xZdq-R-9hHvLF7-spofiwdeK3Jw,5466 +deepspeed/compile/passes/zero1_compile.py,sha256=UHM5rtq1JX6Rjj0asbF7Nt9EUy25pvalseM4xXd7QsY,2323 +deepspeed/compile/passes/zero3_compile.py,sha256=PzrtTW-oDyEWQb_J38F3cAzIsRvU0w100SWv-FnB8L8,7645 +deepspeed/compile/patch_compiled_func.py,sha256=ijs5Yo7IkLve0j5Hpwe3QxiiTL1AEhOrlGCW7GiAjVo,2856 +deepspeed/compile/patch_fake_tensor.py,sha256=1d-sXFu63HF2vCPqA7Wg3rVvUupeQjezrRlqeMOaYxU,2010 +deepspeed/compile/profilers/__init__.py,sha256=OSidjoyeb6NwY7vBzdvPEJHDPpS08WRUbR-m8HFLyXE,957 +deepspeed/compile/profilers/__pycache__/__init__.cpython-310.pyc,, +deepspeed/compile/profilers/__pycache__/comm_profile.cpython-310.pyc,, +deepspeed/compile/profilers/__pycache__/graph_profile.cpython-310.pyc,, +deepspeed/compile/profilers/comm_profile.py,sha256=xM3vbFB7gYPdyVqTicuEqOCB4mW89TCR81j6kRVVbew,5103 +deepspeed/compile/profilers/graph_profile.py,sha256=UDzJGK7ENwoDzpB2LepW0Aa--CHmF4I3Ow5D5rdoSrk,11243 +deepspeed/compile/util.py,sha256=Zirqj_4obWpWc-wPi_Hrn5SU7UZ11vBIiSALu4WHKCE,15586 +deepspeed/compression/__init__.py,sha256=s3fNEEJJdLa4D4rEwtraP5axoSqpxxmh6LAIvyzfUTM,243 +deepspeed/compression/__pycache__/__init__.cpython-310.pyc,, +deepspeed/compression/__pycache__/basic_layer.cpython-310.pyc,, +deepspeed/compression/__pycache__/compress.cpython-310.pyc,, +deepspeed/compression/__pycache__/config.cpython-310.pyc,, +deepspeed/compression/__pycache__/constants.cpython-310.pyc,, +deepspeed/compression/__pycache__/helper.cpython-310.pyc,, +deepspeed/compression/__pycache__/scheduler.cpython-310.pyc,, +deepspeed/compression/__pycache__/utils.cpython-310.pyc,, +deepspeed/compression/basic_layer.py,sha256=oZvNusdxqnlhM-RsaALf5Q-4m2kQLcKAUUMuIKBtXec,36047 +deepspeed/compression/compress.py,sha256=DueechYg3bDNAIn2XW0L9nv8qN81acTPwimvD4usE_8,11886 +deepspeed/compression/config.py,sha256=5TTZNLB_GDULXEf704-dbNJo0ILL47DNVWHWiMGio8Q,25067 +deepspeed/compression/constants.py,sha256=Fc2681jhRfm0ADzNjFKjhYIPSXAI29gqeQSGW0FxLoA,5569 +deepspeed/compression/helper.py,sha256=lEFpurhyU7-yKWgO-wp1RojHECkqR8qJ16KrT4Hl06U,14637 +deepspeed/compression/scheduler.py,sha256=QwnyBtRhkEkufjyhns-Riz_-GBdUePzuI8UEQJOu7Ho,8112 +deepspeed/compression/utils.py,sha256=vLWAEOQHa1w61zmMyFQ3dwxbm-5kM_S-zOI_eQhYx5E,7818 +deepspeed/constants.py,sha256=K5l2jzMTXYfqyUBbO9QIMbe_1TFMxdJ9LJDEQ6HJLRY,867 +deepspeed/elasticity/__init__.py,sha256=Cmi3gEN48VYdPSg4zLOJ1KGKP2G3Xeruz9TykgC13m4,383 +deepspeed/elasticity/__pycache__/__init__.cpython-310.pyc,, +deepspeed/elasticity/__pycache__/config.cpython-310.pyc,, +deepspeed/elasticity/__pycache__/constants.cpython-310.pyc,, +deepspeed/elasticity/__pycache__/elastic_agent.cpython-310.pyc,, +deepspeed/elasticity/__pycache__/elasticity.cpython-310.pyc,, +deepspeed/elasticity/__pycache__/utils.cpython-310.pyc,, +deepspeed/elasticity/config.py,sha256=OCmF8fP8_dL3LkKiEJDtOREnJB3NYQuCaK48qRs-eFA,4703 +deepspeed/elasticity/constants.py,sha256=5Ynz57XbIFxeFkhC3JwanR-Dw0-U7qrsqKIH9oQo-Hw,2454 +deepspeed/elasticity/elastic_agent.py,sha256=KrPIykg6Q4K6BrjkgWmj0PRkgGHNuOb5vvgNjf7XZwk,8009 +deepspeed/elasticity/elasticity.py,sha256=oGJXhEj-VVdT2iUa-ei_gHXAfhP5e_VgtUlw5N500mU,17361 +deepspeed/elasticity/utils.py,sha256=-mbP_VX-SSstZKrEp0hkFwezql9pkAEvxx3BxMO2-ic,347 +deepspeed/env_report.py,sha256=0ZVv7RaHSaQIOJKWG5dqs6OfFalx6adq3NFZwB3nVIs,7491 +deepspeed/git_version_info.py,sha256=IZlPdw3IO5LiSJQ5ECQBVNnIj9TL3ezz-EEYfSR54ag,1075 +deepspeed/git_version_info_installed.py,sha256=orsv9tbsfTxvqxsz_A-2vF6h9rwkrSj6Mr5KACBFISY,729 +deepspeed/inference/__init__.py,sha256=ZV8FRXVTJ3IGIDnyXJgJjRlCbB2jH_KJ1hWoTS4l_y0,267 +deepspeed/inference/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/__pycache__/config.cpython-310.pyc,, +deepspeed/inference/__pycache__/engine.cpython-310.pyc,, +deepspeed/inference/config.py,sha256=zBiw9UvgAQ79FfauGJPLpXwXgtxrrBMUwUdokM-TEww,11283 +deepspeed/inference/engine.py,sha256=2ndGZLN0dHv2RDDeaWj0do5o5OLrzutfqZMPDO6DsIA,30250 +deepspeed/inference/quantization/__init__.py,sha256=4I9UpQ5vMRU5SYSF_dW9FJDEnBq4m_0SuwtVQ92lGaA,95 +deepspeed/inference/quantization/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/quantization/__pycache__/layers.cpython-310.pyc,, +deepspeed/inference/quantization/__pycache__/quantization.cpython-310.pyc,, +deepspeed/inference/quantization/__pycache__/quantization_context.cpython-310.pyc,, +deepspeed/inference/quantization/__pycache__/utils.cpython-310.pyc,, +deepspeed/inference/quantization/layers.py,sha256=s62KAh9pysdpMzV3KuaFeOdrezxLSsPcZi-Fmsj0w84,5729 +deepspeed/inference/quantization/quantization.py,sha256=-r1gbQC0AZe3umfsjla_2lp6opWrfI-0jfs6KtOaJ8o,4396 +deepspeed/inference/quantization/quantization_context.py,sha256=0IV04DTAk8PiOi07TGgABxFMkrKM4QFOqzRaT-RZO7w,514 +deepspeed/inference/quantization/utils.py,sha256=qu0HJHOSz020DT1UuMUo45c1IyARecLl_cCoMcDuf9c,11933 +deepspeed/inference/v2/__init__.py,sha256=9LrBSo2xdUl33MPzLedqmI13rnQOJJO-eT6fVsrt61k,283 +deepspeed/inference/v2/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/__pycache__/allocator.cpython-310.pyc,, +deepspeed/inference/v2/__pycache__/config_v2.cpython-310.pyc,, +deepspeed/inference/v2/__pycache__/engine_factory.cpython-310.pyc,, +deepspeed/inference/v2/__pycache__/engine_v2.cpython-310.pyc,, +deepspeed/inference/v2/__pycache__/inference_parameter.cpython-310.pyc,, +deepspeed/inference/v2/__pycache__/inference_utils.cpython-310.pyc,, +deepspeed/inference/v2/__pycache__/logging.cpython-310.pyc,, +deepspeed/inference/v2/__pycache__/scheduling_utils.cpython-310.pyc,, +deepspeed/inference/v2/allocator.py,sha256=A-yTKojaNr_O8COkDIFagTl4dd9aYJRIMtR7g8TIvJ4,1182 +deepspeed/inference/v2/checkpoint/__init__.py,sha256=6y_8HbNnkff-I3LYNyzZQ3DVDcy8CwjQAYykp_JiCw8,252 +deepspeed/inference/v2/checkpoint/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/checkpoint/__pycache__/base_engine.cpython-310.pyc,, +deepspeed/inference/v2/checkpoint/__pycache__/huggingface_engine.cpython-310.pyc,, +deepspeed/inference/v2/checkpoint/__pycache__/in_memory_engine.cpython-310.pyc,, +deepspeed/inference/v2/checkpoint/base_engine.py,sha256=9EmyYEwt1ZE861yuS-ayx9ykG2OzRi9p7_ua4hECMEs,1391 +deepspeed/inference/v2/checkpoint/huggingface_engine.py,sha256=sn6nmRLiOLEVxTCMHRExUbbIjMVavCAFyerbdxssz-M,5997 +deepspeed/inference/v2/checkpoint/in_memory_engine.py,sha256=4tEQQPAWl4BG4FJFdcgnoa8jahMQUtrXtcssTwRUSeo,1487 +deepspeed/inference/v2/config_v2.py,sha256=HT1FCeys_QnuUTNteBLQhJ8Y6Cg4WVW5B1HwAwQSUX4,1389 +deepspeed/inference/v2/engine_factory.py,sha256=ap6CwbdaOvD3IcCOCaqfhAvIRhp0lu0efZCLHTEMST8,6028 +deepspeed/inference/v2/engine_v2.py,sha256=2IiT1buZLPYL66oC8OQdpANhRSFjN-xjvnnCJh9Wqts,10491 +deepspeed/inference/v2/inference_parameter.py,sha256=rPeDJ4h58L4weNeIiFDIUrpVgDLpkQtzihUl_L-rkbg,2788 +deepspeed/inference/v2/inference_utils.py,sha256=V5gryIBXY9kVVkwPWukMMzQKbn0vH2hyDWnDhQ4jWQk,2378 +deepspeed/inference/v2/kernels/__init__.py,sha256=KHPoSEGybfJdU54HoIOCZzeIovB3EsfzaZuHv_i36V4,132 +deepspeed/inference/v2/kernels/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/__pycache__/ds_kernel.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/__init__.py,sha256=72TCKbfeUY6LbxtwVeNKljxXvxFrY-zobANphLq7nVQ,276 +deepspeed/inference/v2/kernels/core_ops/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/bias_activations/__init__.py,sha256=BpRq2zT5tKcsx4e7f8zcnDA5F6wEedmQXSI6idSEoTo,127 +deepspeed/inference/v2/kernels/core_ops/bias_activations/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/bias_activations/__pycache__/bias_activation.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/bias_activations/bias_activation.cpp,sha256=PdSuObELmrSu3lsYgfVEOO9tEEmexqO0an3_pP4nU3k,3084 +deepspeed/inference/v2/kernels/core_ops/bias_activations/bias_activation.h,sha256=q3vJjn6I7s_raM1Gy2TNnrY3nbskDDP9QyWwcDjpk6E,688 +deepspeed/inference/v2/kernels/core_ops/bias_activations/bias_activation.py,sha256=6uu2cWl65xBVWVHfZR9TfLiqrovUJOxOm50y2iZ9Y28,2437 +deepspeed/inference/v2/kernels/core_ops/bias_activations/bias_activation_cuda.cu,sha256=dq_Bwd-UGv8bK3wWA1llNGVUEXO_rAMZInl5PcfpO4Y,4820 +deepspeed/inference/v2/kernels/core_ops/blas_kernels/__init__.py,sha256=ND9hDC4WOLfDXnlqe7FySprn8RkCXvzrwQKwYG8ZFRQ,123 +deepspeed/inference/v2/kernels/core_ops/blas_kernels/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/blas_kernels/__pycache__/blas_linear.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/blas_kernels/blas.h,sha256=YtINP8OfWWiyZEx44zb1yylsXBHGBLdfxcb3xjBUNeI,5164 +deepspeed/inference/v2/kernels/core_ops/blas_kernels/blas_linear.py,sha256=AQ4eOYQ3ejZspCeBHm87Q32f_Lf_ooauZIoVobdD_G0,2023 +deepspeed/inference/v2/kernels/core_ops/blas_kernels/blas_utils.h,sha256=IGCBmqS3jASIg3Trt70BRfl2KIxtF2T8fQ2HxIOUJNs,11400 +deepspeed/inference/v2/kernels/core_ops/core_ops.cpp,sha256=4HbbHAOUmkaolemLrP_nWOdXSIbAP7KGgl1jTU1LMfI,1514 +deepspeed/inference/v2/kernels/core_ops/cuda_layer_norm/__init__.py,sha256=tUarONsuAuUxyfogdH03MC3yebDySSrUEusyIEWNjrg,174 +deepspeed/inference/v2/kernels/core_ops/cuda_layer_norm/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/cuda_layer_norm/__pycache__/cuda_fp_ln_base.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/cuda_layer_norm/__pycache__/cuda_ln.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/cuda_layer_norm/__pycache__/cuda_post_ln.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/cuda_layer_norm/__pycache__/cuda_pre_ln.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/cuda_layer_norm/cuda_fp_ln_base.py,sha256=CmnqS4_iIbPWKbzNCKEv3wnng8UnRtyhSJiEz08cfPI,1302 +deepspeed/inference/v2/kernels/core_ops/cuda_layer_norm/cuda_ln.py,sha256=bp4CvANVf8hQQzyKOIScnR1OU7kB41uBNK5ANIydCvs,854 +deepspeed/inference/v2/kernels/core_ops/cuda_layer_norm/cuda_post_ln.py,sha256=cIJ9l9ey_fAdjuX00ZjooepUgvNEg_bc2ikSiR0MWK0,990 +deepspeed/inference/v2/kernels/core_ops/cuda_layer_norm/cuda_pre_ln.py,sha256=meb0fP8vvu1lFGlMR8iEus7YKlGP6FMlad9FJ7ZPg8A,1225 +deepspeed/inference/v2/kernels/core_ops/cuda_layer_norm/layer_norm.cpp,sha256=cWe1Z6kgQuPsrSTCOO1mWBtFY_CYqXHUt1HqhfiOyNs,4199 +deepspeed/inference/v2/kernels/core_ops/cuda_layer_norm/layer_norm.h,sha256=SCuiMaChAEgmE0t2kwfbqaYXQ1ycU1QXDVDJSHxtQ6o,2157 +deepspeed/inference/v2/kernels/core_ops/cuda_layer_norm/layer_norm_cuda.cu,sha256=NLCh_1U7k5iA5EOmmqISVMzoa47ypMYYZ5UAk_8tvMo,19730 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/__init__.py,sha256=0J6ih4FAYwgMMo0HSRWLTiBimI9hdeAFh9Ui5iwx2NU,123 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/cuda_linear/__pycache__/cuda_linear.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/cuda_linear/cuda_linear.py,sha256=_2OxKHsY9P5mKOW3sT1ZTIy72d3BGpEjt7_Kb2bNsD4,6827 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/include/configs.h,sha256=XPl9PmMyD-OVFksGXN-eK67a-aoxJvjxNEBIXm3q6Fs,5044 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/include/kernel_matmul.cuh,sha256=e92X51umAswaFGMz9ntC58TRD1W33CjVZPS2eVeZFOk,14700 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/include/kernel_reduction.cuh,sha256=GQix8cchlsjHeTajB6oM08OEoEC1Mcp62LEsq3XKnAU,1494 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/include/ptx_cp.async.cuh,sha256=kdZEfq2Akuw1rSJMdR6DQ0k23mbYhRuQ4cKffK9_wUc,2357 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/include/ptx_mma.cuh,sha256=_-OJfWV4dhi9or_B1orbEkxBdZCNiXnmjW5-eNHtmno,5139 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/include/utils_core.cuh,sha256=9RDNZ3HQFo0rBbFgaBJpM7JzxXQ4fYiUNdnlsZOfM84,10251 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/include/utils_gmem.cuh,sha256=pyOFBGMAlcDFZx5iSS-HPnXA0Pqwx9oCvrUFV_EkioY,3207 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/include/utils_paralleldequant.cuh,sha256=bqJJIaioGtHe1feD1vIjjWS1OpKGTrau1h03cCdkiaI,4057 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/include/weight_prepacking.h,sha256=JTrsVJdRovts8Q7bG7MlCzsVyKL_R76moU-zyakMPGI,8896 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/linear_kernels.cpp,sha256=XIFk-nXPumydZOOpupiRPrqgN6wbPRMr86u8_AAqcnM,8892 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/linear_kernels.h,sha256=i97f3j3gOIiFK4Ijoiu05U9aj0S4p2NfhBIIYe-hAbM,828 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/linear_kernels_cuda.cu,sha256=l5rmu_QnbM1VNmao97uVySwP2sOGjjF919pX18_KXwA,13947 +deepspeed/inference/v2/kernels/core_ops/cuda_linear/linear_kernels_cuda.h,sha256=ON5qqFROLBv392C3nwM0adOlKtCHsjO59yYUgGQwz_A,1669 +deepspeed/inference/v2/kernels/core_ops/cuda_rms_norm/__init__.py,sha256=S3Q1i1iwyLp6kWWJjD2kTinhJQ_ND4gKHejhuxQ8-74,171 +deepspeed/inference/v2/kernels/core_ops/cuda_rms_norm/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/cuda_rms_norm/__pycache__/rms_norm.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/cuda_rms_norm/__pycache__/rms_norm_base.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/cuda_rms_norm/__pycache__/rms_pre_norm.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/cuda_rms_norm/rms_norm.cpp,sha256=XqvW2ObN0dkRQIsqtWRGl_JYxkFjKAi-zjFMeEZVa3M,5873 +deepspeed/inference/v2/kernels/core_ops/cuda_rms_norm/rms_norm.h,sha256=FUf2hMir5TnFA6iU1Rx48DlnSYX__Yl0jkiNWARdREA,978 +deepspeed/inference/v2/kernels/core_ops/cuda_rms_norm/rms_norm.py,sha256=OkC86DHLQ03zshmFNUQO9r1mZ4mWD3gN0505IvzC1NE,770 +deepspeed/inference/v2/kernels/core_ops/cuda_rms_norm/rms_norm_base.py,sha256=cbZXvZSXRaD1MoTokRSRuw1anp_FXdGzg0moAxasQ1I,1311 +deepspeed/inference/v2/kernels/core_ops/cuda_rms_norm/rms_norm_cuda.cu,sha256=GrWivwyCSZ8gh1zaiJoKHh8sTcq8aot5m7XKx-Auw1A,10216 +deepspeed/inference/v2/kernels/core_ops/cuda_rms_norm/rms_pre_norm.py,sha256=qzME0FbrgV9IXbGCqIA0IP91rfFyRiTdiXwwQ39SufM,1204 +deepspeed/inference/v2/kernels/core_ops/gated_activations/__init__.py,sha256=2mIcBil301zqLsisa-B2C2EnTgQ2Zaz2QFRxPTD8eUE,128 +deepspeed/inference/v2/kernels/core_ops/gated_activations/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/gated_activations/__pycache__/gated_activation.cpython-310.pyc,, +deepspeed/inference/v2/kernels/core_ops/gated_activations/gated_activation.py,sha256=XQXwTY3PN0qWIdZMBWh8De2lEGtN8syw7ItH_uN319g,2755 +deepspeed/inference/v2/kernels/core_ops/gated_activations/gated_activation_kernels.cpp,sha256=aog8VhJGL0F4bwUOZTR232KTgRJRavFFk3Xj_55shLk,3521 +deepspeed/inference/v2/kernels/core_ops/gated_activations/gated_activation_kernels.h,sha256=DpAhhx6lO3TNA8eid6Tct9qfDrNFweHRypmn4serMW0,776 +deepspeed/inference/v2/kernels/core_ops/gated_activations/gated_activation_kernels_cuda.cu,sha256=0uLVwa_H3oSJczKK61AZCbBIUZF1mts1QGIM3yKUI6k,6080 +deepspeed/inference/v2/kernels/cutlass_ops/__init__.py,sha256=hpTBFM7draPD0dwxTHvGYgAdic-ysFMG6_XEr8x6V5c,146 +deepspeed/inference/v2/kernels/cutlass_ops/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/cutlass_ops/cutlass_ops.cpp,sha256=v6-PzCFZnmxFPXJm-pooeUISCq2V0S4E4_PAvg7oA6w,483 +deepspeed/inference/v2/kernels/cutlass_ops/mixed_gemm/__init__.py,sha256=leIZbZ0995YwKwql6mU54HN_DJ1iBe67Cq_GdlXSMAY,122 +deepspeed/inference/v2/kernels/cutlass_ops/mixed_gemm/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/cutlass_ops/mixed_gemm/__pycache__/mixed_gemm.cpython-310.pyc,, +deepspeed/inference/v2/kernels/cutlass_ops/mixed_gemm/mixed_gemm.cu,sha256=fDh2z9IwBffz5AI1NltOo4tgC2a2c0bwZ-leLahnR2k,4131 +deepspeed/inference/v2/kernels/cutlass_ops/mixed_gemm/mixed_gemm.h,sha256=laUi-GFh6qbLo0Ywiz_h7TtNIxroXSFwPQKc6yBGhE8,410 +deepspeed/inference/v2/kernels/cutlass_ops/mixed_gemm/mixed_gemm.py,sha256=gZ7ljMs2f20rcL1624vIhiLg_yemxkrYYFDoX-K1M3Y,2674 +deepspeed/inference/v2/kernels/cutlass_ops/mixed_gemm/mixed_gemm_api.h,sha256=soDC6vGW7UCYrnfw1EftpmOz6BuOi27Ihyi8J2Cfw2Q,1570 +deepspeed/inference/v2/kernels/cutlass_ops/moe_gemm/__init__.py,sha256=RRc_fJh0o2_l9wVb7Czf2GqTxLzvS6Rj_zeYUGzRjjI,150 +deepspeed/inference/v2/kernels/cutlass_ops/moe_gemm/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/cutlass_ops/moe_gemm/__pycache__/mixed_moe_gemm.cpython-310.pyc,, +deepspeed/inference/v2/kernels/cutlass_ops/moe_gemm/__pycache__/moe_gemm.cpython-310.pyc,, +deepspeed/inference/v2/kernels/cutlass_ops/moe_gemm/mixed_moe_gemm.py,sha256=pAaowuHqgwtCqv3tyof9jpbPxZxgLdcj1vPM2NgDcx4,2996 +deepspeed/inference/v2/kernels/cutlass_ops/moe_gemm/moe_gemm.cu,sha256=FRZSfEKLH3lHjvdbTbuEr1qGAi1spT6lhTLuIJ_OGuI,8594 +deepspeed/inference/v2/kernels/cutlass_ops/moe_gemm/moe_gemm.h,sha256=hdqzk6YOF68wO1DBVWDdtX5z-MIGHdtekijLU8z1cHM,740 +deepspeed/inference/v2/kernels/cutlass_ops/moe_gemm/moe_gemm.py,sha256=zX6btY3JLxP-6mgcjlNLSHlmgHJ2z3ZMH6PV3DQPfeE,2572 +deepspeed/inference/v2/kernels/cutlass_ops/moe_gemm/moe_gemm_api.h,sha256=RWmGN2LrcQukrsLoYaqykymCQJ1onmubdqXrHHgx2Is,1839 +deepspeed/inference/v2/kernels/cutlass_ops/shared_resources/weight_variant.h,sha256=1Mr74mw0SDFt0BsMgWiLvYXXaMWmhIgCUz1MZlWXYMs,278 +deepspeed/inference/v2/kernels/ds_kernel.py,sha256=R0n_uBCLCGAGlJa7lvep4cNF0DBxkD6TDKDFNAWpBpU,961 +deepspeed/inference/v2/kernels/includes/activation_type.h,sha256=WkFfkL5mgLyYAZimLUouThCkyA2aiF9LQlh1q206L9o,264 +deepspeed/inference/v2/kernels/includes/conversion_utils.h,sha256=augq8Zf-t46MaxWh0ryFb46_Guxud0k4kIK-h93rI8Y,12379 +deepspeed/inference/v2/kernels/includes/ds_kernel_utils.h,sha256=OKOMT46XItRnkrzqKHHK9n_ewxHh6AN-Uhv4aKixBm4,1297 +deepspeed/inference/v2/kernels/includes/memory_access_utils.h,sha256=wqpAqnfmSD5Fx15qwsnY63VCL6tYyJaChCqLxakM10w,33966 +deepspeed/inference/v2/kernels/includes/reduction_utils.h,sha256=189h93WF77CblgWonfW5eQqoTeJnb-Rt2Ov2fZ6mO-E,22352 +deepspeed/inference/v2/kernels/ragged_ops/__init__.py,sha256=8C6TomumccM_-p-iiyPyjZnDLPjTdQS8E4ydGQGNLu4,324 +deepspeed/inference/v2/kernels/ragged_ops/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/atom_builder/__init__.py,sha256=djM0DTjGkr4vvNuhRIWV-2OI7lroepRMBiHIJtrcqTA,124 +deepspeed/inference/v2/kernels/ragged_ops/atom_builder/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/atom_builder/__pycache__/atom_builder.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/atom_builder/atom_builder.cpp,sha256=AjIgvgoS7PUYGt4MW7oFu8QEcQvm3J16tyerW-dx8cI,2051 +deepspeed/inference/v2/kernels/ragged_ops/atom_builder/atom_builder.h,sha256=i232uHXwHjh2EaAF3709YNcTQ29uVDI2GQOs6GnYAsQ,732 +deepspeed/inference/v2/kernels/ragged_ops/atom_builder/atom_builder.py,sha256=QgVnamgI1qG8xjzu-9i5sdBccOuhgbTeAtufUtLNdog,1744 +deepspeed/inference/v2/kernels/ragged_ops/blocked_flash/__init__.py,sha256=696rc8-r9nWCnfSG8UZ_fagI28Nerwq-nZCDvIIXPRs,125 +deepspeed/inference/v2/kernels/ragged_ops/blocked_flash/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/blocked_flash/__pycache__/blocked_flash.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/blocked_flash/attention_atom.h,sha256=-ITylnAQx_20rXiWdPPldb149cGG-fNQUqOg8iHVugU,1062 +deepspeed/inference/v2/kernels/ragged_ops/blocked_flash/blocked_flash.cpp,sha256=qaNovmlpqw7Cw_Y1ArGmH-7fB1nPQd0idvmC4nDR9pw,3912 +deepspeed/inference/v2/kernels/ragged_ops/blocked_flash/blocked_flash.h,sha256=io7L6krERzOgiBIKNk3zwEppMDSvWtOqCY9nH35dB_U,459 +deepspeed/inference/v2/kernels/ragged_ops/blocked_flash/blocked_flash.py,sha256=-aU8A7FbwrRqJPSHZEOef6Y3_PRndGKfCFT1c-3S1DQ,3830 +deepspeed/inference/v2/kernels/ragged_ops/blocked_flash/flash.h,sha256=VJKtKOCufGTGt8ZGuPXfsA-dzsi87cqNTYX7pbqHKH4,1931 +deepspeed/inference/v2/kernels/ragged_ops/embed/__init__.py,sha256=y4HCIAaLuHexvTBcNppgiXqo76vHb_pAahL75kQBKY0,137 +deepspeed/inference/v2/kernels/ragged_ops/embed/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/embed/__pycache__/embed.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/embed/embed.cpp,sha256=iI-rjkQlCGnujjVzY1h8e6ccO24G7eVxwPLY5t_HKxY,4840 +deepspeed/inference/v2/kernels/ragged_ops/embed/embed.cuh,sha256=C5ZS1W9lnEDPBmWsuoJpRP-pgCYwLxqTK_OI-DQrT04,956 +deepspeed/inference/v2/kernels/ragged_ops/embed/embed.h,sha256=XbqCgqlx16bRtRLljUstzjssRJIGx7AixbbaofhILl8,700 +deepspeed/inference/v2/kernels/ragged_ops/embed/embed.py,sha256=6lxDG2ymTyx2yGZ_9jDFyvcchbtD1iwKr2icLgHtmxE,3011 +deepspeed/inference/v2/kernels/ragged_ops/embed/embed_cuda.cu,sha256=5JfvhMBcKAPXQbF4WvE3Ljmolom8ArjvkNtprx1SaKo,6055 +deepspeed/inference/v2/kernels/ragged_ops/includes/top_k_utils.h,sha256=T0LJvEtOulSPO1r0YA4jEHW4l-3XSYyS8nuI6Ok2F6E,782 +deepspeed/inference/v2/kernels/ragged_ops/linear_blocked_kv_rotary/__init__.py,sha256=ou-mEvP9FMj89HvxswZbtTvS_T8Gcy3zf16jPcb_Urw,208 +deepspeed/inference/v2/kernels/ragged_ops/linear_blocked_kv_rotary/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/linear_blocked_kv_rotary/__pycache__/blocked_kv_rotary.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/linear_blocked_kv_rotary/__pycache__/blocked_trained_kv_rotary.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/linear_blocked_kv_rotary/__pycache__/linear_blocked_kv_copy.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/linear_blocked_kv_rotary/blocked_kv_rotary.cpp,sha256=qUdVHb-2UMnlH8Q6zZJLUW69T75AMwfqEZjco2EXbdY,8110 +deepspeed/inference/v2/kernels/ragged_ops/linear_blocked_kv_rotary/blocked_kv_rotary.cuh,sha256=X9w1u75CiGNEOV0uaXAmjX8usEiTXFTZG7EK3ulvZDc,1676 +deepspeed/inference/v2/kernels/ragged_ops/linear_blocked_kv_rotary/blocked_kv_rotary.h,sha256=5RfNKDify3eHIV6GjfqNozT6_lc2shcwNidgrQ6AJD0,2406 +deepspeed/inference/v2/kernels/ragged_ops/linear_blocked_kv_rotary/blocked_kv_rotary.py,sha256=spVrPPi6tMh31Zjb3l4UooFgJ2YTFq9UJI2f50Mziuk,3142 +deepspeed/inference/v2/kernels/ragged_ops/linear_blocked_kv_rotary/blocked_kv_rotary_cuda.cu,sha256=ztJXnG_ik8nCCJQw2d3P0ScuNmTUPvCKQT1I3L28EzA,19275 +deepspeed/inference/v2/kernels/ragged_ops/linear_blocked_kv_rotary/blocked_trained_kv_rotary.py,sha256=FGU4a0Wcb4NtsbJiLswLnd6-M89yE800UW19v5kFp9Y,3220 +deepspeed/inference/v2/kernels/ragged_ops/linear_blocked_kv_rotary/linear_blocked_kv_copy.py,sha256=mAa-291Bzc06yNGZVgsYPhrpq7dVoYDkWDHFUr2i1fY,2957 +deepspeed/inference/v2/kernels/ragged_ops/logits_gather/__init__.py,sha256=-6rVFXUnZ3U4MBA1VNuKQfhcdjlWgc_xnufXpnFfGyg,125 +deepspeed/inference/v2/kernels/ragged_ops/logits_gather/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/logits_gather/__pycache__/logits_gather.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/logits_gather/logits_gather.cpp,sha256=wItdeUW_dSJzfePXbRKjYeggcfGG_l8yO7OAPXF4IeU,1800 +deepspeed/inference/v2/kernels/ragged_ops/logits_gather/logits_gather.cuh,sha256=OQlrhwuHJIwGfhbozWlM4NKhYoKBtAL-3nc0vqATdZY,624 +deepspeed/inference/v2/kernels/ragged_ops/logits_gather/logits_gather.h,sha256=bRw1hkiDAAqkP9FkNwsL1QZ0UeGSx1_pYHOY4yoTF5k,571 +deepspeed/inference/v2/kernels/ragged_ops/logits_gather/logits_gather.py,sha256=kGPLmxKKqOh2x35QWtCpeXLXxoG15kxmE_Z_-cd7Yv0,2121 +deepspeed/inference/v2/kernels/ragged_ops/logits_gather/logits_gather_cuda.cu,sha256=Dd2HJJHpPlDkt2C91667Fn1GVuc4jonsIW8s1vPKhHk,3312 +deepspeed/inference/v2/kernels/ragged_ops/moe_gather/__init__.py,sha256=K3BZ4_CDJGkNmvFCeYvIEBY2DRMWqq5CzCL7HAStX8Q,122 +deepspeed/inference/v2/kernels/ragged_ops/moe_gather/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/moe_gather/__pycache__/moe_gather.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/moe_gather/moe_gather.cpp,sha256=zWULPg_6e15LzTk5q2DJJcsgwT4zoLhTNIc0ClJf73w,2392 +deepspeed/inference/v2/kernels/ragged_ops/moe_gather/moe_gather.cuh,sha256=orang8iBcPNQ0BjFTeiBWNX09UjDq82Sh8tyye-5xyU,706 +deepspeed/inference/v2/kernels/ragged_ops/moe_gather/moe_gather.h,sha256=MfJuJ2rg_ba6KvIm1LNbTTHMBH8MYQHgd7JeYJSG9Q8,561 +deepspeed/inference/v2/kernels/ragged_ops/moe_gather/moe_gather.py,sha256=31zC0Sb3caA12JdhT7qJoawCUTkKB5PjZpFupQI6FvE,2265 +deepspeed/inference/v2/kernels/ragged_ops/moe_gather/moe_gather_cuda.cu,sha256=oy_bzQhyMfmjwiXag0u4YKjyKHaLXMDWYHlJQnQZNWM,6679 +deepspeed/inference/v2/kernels/ragged_ops/moe_scatter/__init__.py,sha256=e3cxpdOKqIBYpxvNmMnj6Ms883dqdsl4B_fXdqTnlnM,123 +deepspeed/inference/v2/kernels/ragged_ops/moe_scatter/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/moe_scatter/__pycache__/moe_scatter.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/moe_scatter/moe_scatter.cpp,sha256=ywodf_RYumTot90MnTLb4o1h7HDWgiFuVTuhHwxNooE,2858 +deepspeed/inference/v2/kernels/ragged_ops/moe_scatter/moe_scatter.cuh,sha256=M6t_ydSLYKsekCym5m07AGDu8-U_qVG0hLuVH_nrO-A,772 +deepspeed/inference/v2/kernels/ragged_ops/moe_scatter/moe_scatter.h,sha256=pBdy99sow7TKoB3eV3gdPbCB4_pGNx48K-Eo8C8XjHs,701 +deepspeed/inference/v2/kernels/ragged_ops/moe_scatter/moe_scatter.py,sha256=7c_jFxhoJINWl57LIhFZwvlr08W8Nbdk7ULSFmC8Yqc,2504 +deepspeed/inference/v2/kernels/ragged_ops/moe_scatter/moe_scatter_cuda.cu,sha256=US5_PjVcIqYcqR2DDKXUIJAMfdXKI0ElfbNCUmLscbw,8070 +deepspeed/inference/v2/kernels/ragged_ops/ragged_helpers/ragged_dtypes.h,sha256=OVEob4D9LFPkyxA4PscOrGV5BzBalfiMrTrkcifCvL0,940 +deepspeed/inference/v2/kernels/ragged_ops/ragged_helpers/ragged_kernel_helpers.cpp,sha256=QaWqjTz4kXXR804xXlgNBTOxtOtLSP3NcGDNHuFqqQI,1149 +deepspeed/inference/v2/kernels/ragged_ops/ragged_helpers/ragged_kernel_helpers.h,sha256=_7Xb4l1mfYbSz7i2HAZ8TqQSLJy6TBBC51OMaarJXfM,563 +deepspeed/inference/v2/kernels/ragged_ops/ragged_ops.cpp,sha256=Bnku0tlvBSyPIwOtx7-kvIGg3OPGmsgjMO72rLABEkQ,1472 +deepspeed/inference/v2/kernels/ragged_ops/top_k_gating/__init__.py,sha256=N0qSPddtzqj_UkISW7rxEVXpdk8a5JItbXBjNAY_FBE,139 +deepspeed/inference/v2/kernels/ragged_ops/top_k_gating/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/top_k_gating/__pycache__/top_k_gating.cpython-310.pyc,, +deepspeed/inference/v2/kernels/ragged_ops/top_k_gating/top_k_gating.cpp,sha256=EBxWZ_Nm2ygSbFtnzI7kGMsl3nZxECXXefAVjMDDVx4,2435 +deepspeed/inference/v2/kernels/ragged_ops/top_k_gating/top_k_gating.cuh,sha256=1U1wNpJPmIhCiSwRi7xeRSR_zCcGnuXbFwBOz_gQplw,751 +deepspeed/inference/v2/kernels/ragged_ops/top_k_gating/top_k_gating.h,sha256=FdryRq5nESxX9HgzKIhnUgwfaoOzLir5O361gf-4g_M,556 +deepspeed/inference/v2/kernels/ragged_ops/top_k_gating/top_k_gating.py,sha256=-xgNLLmp6ea_QIMkGtuYDgt-fLfvq6wPVKgMWQpIHfA,2578 +deepspeed/inference/v2/kernels/ragged_ops/top_k_gating/top_k_gating_cuda.cu,sha256=LCuHhCZhWYGT69EW4mq3sPYr9JafW9Ef1YIScTNCQC0,5130 +deepspeed/inference/v2/logging.py,sha256=H4Dp4vKNi42HZaNgXOAOG-Vki2oC3U3APVQwq3ay8kE,780 +deepspeed/inference/v2/model_implementations/__init__.py,sha256=g4p-wyfxcks_kDpGydBi2Sho_dI46rEfin8-xAydRd4,577 +deepspeed/inference/v2/model_implementations/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/__pycache__/flat_model_helpers.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/__pycache__/inference_model_base.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/__pycache__/inference_policy_base.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/__pycache__/inference_transformer_base.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/__pycache__/layer_container_base.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/__pycache__/parameter_base.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/common_parameters/__init__.py,sha256=_mp6NDvxO-3TM9LiiOTrMBRIweeiRjU6s6L-GpTvX9U,359 +deepspeed/inference/v2/model_implementations/common_parameters/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/common_parameters/__pycache__/attn_output_parameters.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/common_parameters/__pycache__/embedding_parameters.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/common_parameters/__pycache__/invfreq_parameters.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/common_parameters/__pycache__/mlp_parameters.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/common_parameters/__pycache__/moe_parameters.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/common_parameters/__pycache__/norm_parameters.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/common_parameters/__pycache__/qkv_parameters.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/common_parameters/__pycache__/unembed_parameters.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/common_parameters/attn_output_parameters.py,sha256=VwOj8hRbIQQnWfoHVq-vbRK_QLt9f-uimARsTHfot4w,763 +deepspeed/inference/v2/model_implementations/common_parameters/embedding_parameters.py,sha256=k7CMnQ06UchamvQXsdeiTfxAkjpor3qZjTU0cD20hZU,605 +deepspeed/inference/v2/model_implementations/common_parameters/invfreq_parameters.py,sha256=Srz4b0mFK73OBzsw3EK3Aydj21Z1gZjLuyC8qfry6QI,401 +deepspeed/inference/v2/model_implementations/common_parameters/mlp_parameters.py,sha256=fMdVvya2UlX-DJ52QT0Mv8YKFukIAWx4F833ZoI0-IA,3363 +deepspeed/inference/v2/model_implementations/common_parameters/moe_parameters.py,sha256=hTKZ8zIERsvQ6i6zQRWe47qUcJmMAGOHfJinwpY3r04,2548 +deepspeed/inference/v2/model_implementations/common_parameters/norm_parameters.py,sha256=xTaLT5H81-Et8y9FivEaY8N9VQ0XDsvtlV4v1wnFUG4,454 +deepspeed/inference/v2/model_implementations/common_parameters/qkv_parameters.py,sha256=pxBm6EBMNLvbJTPAY9ABjdQ68-oxbY_nWb5CFGDdvdA,4086 +deepspeed/inference/v2/model_implementations/common_parameters/unembed_parameters.py,sha256=-gdVA8UMMvFsd-lSAvCbQp5mQ25j1sT5f7KC2mSVY_A,651 +deepspeed/inference/v2/model_implementations/falcon/__init__.py,sha256=IVA-W-wyht49JDQt7MTTsBMN4dHAAQxWPEULVVwkoVY,129 +deepspeed/inference/v2/model_implementations/falcon/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/falcon/__pycache__/container.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/falcon/__pycache__/model.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/falcon/__pycache__/policy.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/falcon/container.py,sha256=TNdqevl_yh4g06l80i3RJC88dExK2bfd_22NSA-yz0M,4479 +deepspeed/inference/v2/model_implementations/falcon/model.py,sha256=ELmk5fmti6cUmUqYRcsnykS746An6eEibPf0G3IQsgs,7768 +deepspeed/inference/v2/model_implementations/falcon/policy.py,sha256=oFyIvfzFEistRJCzXS-wPtFfUXDPqt9ANhPa7ywrKSQ,1342 +deepspeed/inference/v2/model_implementations/flat_model_helpers.py,sha256=XT6oMqJAfZhYLCr5KrPBxSFVW-WKnFjOi3FfEElshP4,9772 +deepspeed/inference/v2/model_implementations/inference_model_base.py,sha256=3YVtQafVIrZMI-kZadRKmQklafI5fjc38GTdTZeM3jQ,10083 +deepspeed/inference/v2/model_implementations/inference_policy_base.py,sha256=XKH3GcOtP8WAeAopwuQACflJA3cvX4F2MGybrIqcBPc,9481 +deepspeed/inference/v2/model_implementations/inference_transformer_base.py,sha256=SgXRyFYWj31DJMbDpgo06nEyAJLD-w3qKJF1xbAm9Jo,23592 +deepspeed/inference/v2/model_implementations/layer_container_base.py,sha256=qvvGjZ0aL6Ah4MDtjpowV76DKSpI8VuZyY_4hZRbB1o,15471 +deepspeed/inference/v2/model_implementations/llama_v2/__init__.py,sha256=J8aLmbt1KDVL_LOtSqaP-xACSxlgIDjeaIA9R-Grsgs,129 +deepspeed/inference/v2/model_implementations/llama_v2/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/llama_v2/__pycache__/container.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/llama_v2/__pycache__/model.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/llama_v2/__pycache__/policy.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/llama_v2/container.py,sha256=blAioiSrdB9kH0zHFV0ZO1kWChv6-geaZW1ivEaSQx8,2664 +deepspeed/inference/v2/model_implementations/llama_v2/model.py,sha256=YmauqGYxtx2o_apo28aYvEhxi3SmYYvLm1IoEE7tGgs,7515 +deepspeed/inference/v2/model_implementations/llama_v2/policy.py,sha256=A1z_j19nj8yOURNNC29DeSyMatAjzuPUtcnIz09zZII,1148 +deepspeed/inference/v2/model_implementations/mistral/__init__.py,sha256=rzXGZBulBvo8ZNt7E5z1mSi8OlN1edNDTFlLCMCqo3M,130 +deepspeed/inference/v2/model_implementations/mistral/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/mistral/__pycache__/container.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/mistral/__pycache__/model.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/mistral/__pycache__/policy.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/mistral/container.py,sha256=0THLABZ6g4W4H_Kf2INGnyVrXHroT2uveKr-F2deP88,2784 +deepspeed/inference/v2/model_implementations/mistral/model.py,sha256=QhDoAoi2aMguJf4GJXqNyivqRKAjJLUwf5EsVgYHl38,7350 +deepspeed/inference/v2/model_implementations/mistral/policy.py,sha256=ceBkx-EOWH6UmcosmmhcE7anO5cDMaPGhLInK-x3PBA,1056 +deepspeed/inference/v2/model_implementations/mixtral/__init__.py,sha256=ywKUfNNkXZuDaMa9NT05H3MkYJQgiCWl3eM8kZMkhJE,130 +deepspeed/inference/v2/model_implementations/mixtral/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/mixtral/__pycache__/container.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/mixtral/__pycache__/model.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/mixtral/__pycache__/policy.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/mixtral/container.py,sha256=YMdy5flaitAQHdy71VBCi7n0vQBfWcPRyXTfZeABchA,1658 +deepspeed/inference/v2/model_implementations/mixtral/model.py,sha256=smAqT1nkiFmn2ipNNvJqd2BiIHHprI5O7iWKOFsKM4M,9096 +deepspeed/inference/v2/model_implementations/mixtral/policy.py,sha256=u1Am_kWG7UvEt8A5V4lTn_BjUZ4eSlnTB32R5agIYy4,1057 +deepspeed/inference/v2/model_implementations/opt/__init__.py,sha256=WeJqbtuxqlXaqif4F_4KpZnGBCwXwKEn0OwUZOcxmpo,126 +deepspeed/inference/v2/model_implementations/opt/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/opt/__pycache__/container.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/opt/__pycache__/model.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/opt/__pycache__/policy.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/opt/container.py,sha256=1jIU2Icv-lSEi_XMhaS4jGUVWFfbP-B2t2xf52eQVwY,3415 +deepspeed/inference/v2/model_implementations/opt/model.py,sha256=3FFAoq-iBo_F5ep0guXKXk6d5BnNOx0QZFiv2VILScs,7262 +deepspeed/inference/v2/model_implementations/opt/policy.py,sha256=9cqEPs_pM9gdzsBegbXJ7GHEgHfmD6n_AaRpjMH1yGE,1066 +deepspeed/inference/v2/model_implementations/parameter_base.py,sha256=FioZn7e4Gw2MUq_ov4popyXxD-dle2X0r7dYagAXmvg,9026 +deepspeed/inference/v2/model_implementations/phi/__init__.py,sha256=jKL7LDi4qrR3X3S6rcFpkls2K0wXVV6Qb3mIkiOtRKk,126 +deepspeed/inference/v2/model_implementations/phi/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/phi/__pycache__/containers.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/phi/__pycache__/model.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/phi/__pycache__/policy.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/phi/containers.py,sha256=L1S3qe2iHCv73EtWzb-IPn_dg8rxzwUn27HPXMatmCg,3197 +deepspeed/inference/v2/model_implementations/phi/model.py,sha256=b12uXjTpbHy_0EklvawD67KxwkrV-ooYSSdjNxGbz58,7251 +deepspeed/inference/v2/model_implementations/phi/policy.py,sha256=FIcCwm7nwUQOoFT1yGV_OTFd2xldUWmL_TqmpjtjUgU,1175 +deepspeed/inference/v2/model_implementations/phi3/__init__.py,sha256=97ojikdkWIzS9unY8jgler8jl3vaOBjGXpifKCtPXxY,127 +deepspeed/inference/v2/model_implementations/phi3/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/phi3/__pycache__/containers.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/phi3/__pycache__/model.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/phi3/__pycache__/policy.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/phi3/containers.py,sha256=hn-rEguXEgYbcu1GSYH7Pc4e6bOFTBQeEbGomMhmG0Y,2464 +deepspeed/inference/v2/model_implementations/phi3/model.py,sha256=0MtLieFL5uPP_eHcKdsAe9Q48ONQIWtPEYuB6OND5wQ,7296 +deepspeed/inference/v2/model_implementations/phi3/policy.py,sha256=ytHLbsKAqQN27Lku715CNU5bMg71EohF5W2-4P7v0lU,1033 +deepspeed/inference/v2/model_implementations/qwen/__init__.py,sha256=1TgzvNCP0loWbohqWRMH9NXCKKjXVVagBtatz_NCHc8,127 +deepspeed/inference/v2/model_implementations/qwen/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/qwen/__pycache__/container.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/qwen/__pycache__/model.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/qwen/__pycache__/policy.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/qwen/container.py,sha256=4fnYCNY92cDw116OyEVhmp_fuFtfeeyq4ZG4T5NHVi0,2362 +deepspeed/inference/v2/model_implementations/qwen/model.py,sha256=cCoVWHR2t_9LrLSwYE2LTCJa6nxXj9-l-Qr5q819Iq0,8112 +deepspeed/inference/v2/model_implementations/qwen/policy.py,sha256=0nP7dboH8FSnh6xvImRKK0gVWkw2iOTImwCJw73V-OA,1066 +deepspeed/inference/v2/model_implementations/qwen_v2/__init__.py,sha256=s0OcwPN9W24-8vYSQ8kztzky9jEwOvER9aMR60Xejuk,128 +deepspeed/inference/v2/model_implementations/qwen_v2/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/qwen_v2/__pycache__/container.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/qwen_v2/__pycache__/model.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/qwen_v2/__pycache__/policy.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/qwen_v2/container.py,sha256=HpDpzng1cMtevhB-XykEKvmPPXENaDRV6BiLVYM7wjo,2817 +deepspeed/inference/v2/model_implementations/qwen_v2/model.py,sha256=EENwSZm-DZRQrv64UCLoEHop_U8xZ9yVejtboPzri_8,8029 +deepspeed/inference/v2/model_implementations/qwen_v2/policy.py,sha256=U9rMVvegwkKMqw6SoFXGZXobYTrjfpfySUryrpF0TDs,1140 +deepspeed/inference/v2/model_implementations/qwen_v2_moe/__init__.py,sha256=nxhwmjwG8giEHatkVAzPoQDEzgLgXCOcxluBG7Y4cn8,131 +deepspeed/inference/v2/model_implementations/qwen_v2_moe/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/qwen_v2_moe/__pycache__/container.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/qwen_v2_moe/__pycache__/model.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/qwen_v2_moe/__pycache__/policy.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/qwen_v2_moe/container.py,sha256=QK62BZaTWjQACeHUJklh_qVJgRbX_N9WGhaOAtY-8dQ,4087 +deepspeed/inference/v2/model_implementations/qwen_v2_moe/model.py,sha256=bcL2sE2G5mHIPk1EZHT3vwFqNEiaBNmosSsKm1d1bjY,13516 +deepspeed/inference/v2/model_implementations/qwen_v2_moe/policy.py,sha256=k_HNyspBw8UG9mjCnCilY8khGf5dp8TI_74Du7eSwCM,1064 +deepspeed/inference/v2/model_implementations/sharding/__init__.py,sha256=M0NsMjUqO0Rj0PAPm2lyRepH5lc3uFQswNM5Gr3QAcI,247 +deepspeed/inference/v2/model_implementations/sharding/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/sharding/__pycache__/attn.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/sharding/__pycache__/attn_out.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/sharding/__pycache__/embedding.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/sharding/__pycache__/mlp.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/sharding/__pycache__/qkv.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/sharding/__pycache__/types.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/sharding/__pycache__/unembed.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/sharding/__pycache__/utils.cpython-310.pyc,, +deepspeed/inference/v2/model_implementations/sharding/attn.py,sha256=QAapGaB4Bp1k9xI53-QhhyKFoaeIPandzI1qnPT2rxE,2376 +deepspeed/inference/v2/model_implementations/sharding/attn_out.py,sha256=MWf6O0P55Am0x947474ZqGcOEuZn2nGptKcNPCxfS0c,4909 +deepspeed/inference/v2/model_implementations/sharding/embedding.py,sha256=mJgZR289D6_ODuTZm4pVsedyJEHaXH2a4ncI8_7YdG0,1270 +deepspeed/inference/v2/model_implementations/sharding/mlp.py,sha256=UIpBwlbU9xDNv1iDbIfcxqPgcAf_WvO42NxtMGyZ1QQ,2895 +deepspeed/inference/v2/model_implementations/sharding/qkv.py,sha256=pQfXbEPeqEZ9eCJwcdSvJsid5jeBJ2jptWhqVaPJSPM,7609 +deepspeed/inference/v2/model_implementations/sharding/types.py,sha256=YqDHnyDLGhy2sf37n-a0eHIIXqcQ8OjHKHQ-rcKAl8E,575 +deepspeed/inference/v2/model_implementations/sharding/unembed.py,sha256=1AmukReQ6SAfFlq2YdiQwJrrvPLVx7fhel7KKCGXXjQ,1625 +deepspeed/inference/v2/model_implementations/sharding/utils.py,sha256=wKlbKIwSyblinL1xupsHDXodws2zfWhxt-flmac1grc,5066 +deepspeed/inference/v2/modules/__init__.py,sha256=oZhP5-VwkgPMtHVFO_9MefXPnSEfcHdQeIXaRg3VOeY,193 +deepspeed/inference/v2/modules/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/modules/__pycache__/ds_module.cpython-310.pyc,, +deepspeed/inference/v2/modules/__pycache__/heuristics.cpython-310.pyc,, +deepspeed/inference/v2/modules/__pycache__/module_registry.cpython-310.pyc,, +deepspeed/inference/v2/modules/configs/__init__.py,sha256=IJ5d0k84imLnM7jh0UcZ9k3_ITPU7YI_6ZoGRpEJcqs,449 +deepspeed/inference/v2/modules/configs/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/modules/configs/__pycache__/attention_configs.cpython-310.pyc,, +deepspeed/inference/v2/modules/configs/__pycache__/embedding_config.cpython-310.pyc,, +deepspeed/inference/v2/modules/configs/__pycache__/linear_config.cpython-310.pyc,, +deepspeed/inference/v2/modules/configs/__pycache__/moe_config.cpython-310.pyc,, +deepspeed/inference/v2/modules/configs/__pycache__/norm_config.cpython-310.pyc,, +deepspeed/inference/v2/modules/configs/__pycache__/unembed_config.cpython-310.pyc,, +deepspeed/inference/v2/modules/configs/attention_configs.py,sha256=IWbmcmCc3CdL1E4FEUhrN9xU9wfbEaswUD8a0cM_D3Y,2834 +deepspeed/inference/v2/modules/configs/embedding_config.py,sha256=PuRrpOWLwtqjZRg9jumeMHuqzzsTcrbCjRxKL2L6xww,1812 +deepspeed/inference/v2/modules/configs/linear_config.py,sha256=fz9m8RIL6NVjbkq06-0WSP4-7IxIdSVQi1nRY4WQdTc,1301 +deepspeed/inference/v2/modules/configs/moe_config.py,sha256=EqX1OLq_QGt4IrSGhAJUXfVvy0MqjZk2E3toeUkb7kA,1289 +deepspeed/inference/v2/modules/configs/norm_config.py,sha256=iTKxfBxWcHs2JiiuxNZU356DGJR-a0OgDW3p5oshedk,839 +deepspeed/inference/v2/modules/configs/unembed_config.py,sha256=2E9XPh8VSRJckwng_DwbQksLMXtWZMFiXUjo5GWhzqI,822 +deepspeed/inference/v2/modules/ds_module.py,sha256=1h8X2TvYvDsUyEUjllPlziT7XOqwRxP02GBagzJw7FU,1856 +deepspeed/inference/v2/modules/heuristics.py,sha256=eRitZ_RX2yg4KOgmjiJaTEDZfH-zauMDIaf3WVQ2rGc,7978 +deepspeed/inference/v2/modules/implementations/__init__.py,sha256=0rEv4jixmtnw_sIe_Y5UKaIEcG7qwALUSzx1GNORTdk,289 +deepspeed/inference/v2/modules/implementations/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/attention/__init__.py,sha256=tNj2_fpavmUWaI5VAkUF0I7k0OxmIlZRNZZl7JIqrR4,157 +deepspeed/inference/v2/modules/implementations/attention/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/attention/__pycache__/dense_blocked_attention.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/attention/dense_blocked_attention.py,sha256=JgS7E2TzKmWylwRYWiYR-F7ldLdg8qhGr8Q9bXSPoCA,7735 +deepspeed/inference/v2/modules/implementations/embedding/__init__.py,sha256=CUiHnEi77BVJmH3aoJOanq5mTJIvEcIo3cHcbkddDHs,144 +deepspeed/inference/v2/modules/implementations/embedding/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/embedding/__pycache__/ragged_embedding.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/embedding/ragged_embedding.py,sha256=zddQlHcpjLe2_dKQUTp_5VfJwTP6L-BRk8HUHD2vPRo,2740 +deepspeed/inference/v2/modules/implementations/linear/__init__.py,sha256=FXT-bP-U4YlbMbBDrgwkpMPTHps3NiztMczMulIgmPU,203 +deepspeed/inference/v2/modules/implementations/linear/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/linear/__pycache__/blas_fp_linear.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/linear/__pycache__/quantized_linear.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/linear/blas_fp_linear.py,sha256=nHZHiLKVcppWyn0hNrQq4sHrS9UYKlHunrwL92DUVkQ,3629 +deepspeed/inference/v2/modules/implementations/linear/quantized_linear.py,sha256=5CQEpEBNq-Tzyt5dH0OK3RzZJUWjAxBieqzuC2OFJgw,7772 +deepspeed/inference/v2/modules/implementations/moe/__init__.py,sha256=XP3Ddz7eBtLvpIGum9u3gzUhl61w0W_jNfopcMI3QRE,143 +deepspeed/inference/v2/modules/implementations/moe/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/moe/__pycache__/cutlass_multi_gemm.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/moe/cutlass_multi_gemm.py,sha256=OzMYYHO7YIT5zOrAqfndOohQgJGwM4oHiPT_1mJU4EU,11126 +deepspeed/inference/v2/modules/implementations/post_norm/__init__.py,sha256=4hE4KqEcl74cOWm1kvexksZwHMK8rk-OQMzJYkdmgQo,141 +deepspeed/inference/v2/modules/implementations/post_norm/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/post_norm/__pycache__/cuda_post_ln.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/post_norm/cuda_post_ln.py,sha256=DNHGpJ2wNC8P6vyyk8M8hDpuXpA7k-ni78W3olRKO18,2086 +deepspeed/inference/v2/modules/implementations/pre_norm/__init__.py,sha256=_yJ5_1apmG5KriWHdy9klvL5H0vFzmLyWnZf1VIrmrA,184 +deepspeed/inference/v2/modules/implementations/pre_norm/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/pre_norm/__pycache__/cuda_pre_ln.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/pre_norm/__pycache__/cuda_pre_rms.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/pre_norm/cuda_pre_ln.py,sha256=VvFlm3Xc6U7VVyfXnFUeA5tsLw4TLhK_LVlKJkN4SmY,2744 +deepspeed/inference/v2/modules/implementations/pre_norm/cuda_pre_rms.py,sha256=WxZeP0B1ghhpOiMIHig4vb0ou0q7jkiLZFfbbdREXtQ,3245 +deepspeed/inference/v2/modules/implementations/unembed/__init__.py,sha256=ve_166eLGPGfIncTIvEvqoFGKi2pvtgvSLVVvTPXpHQ,140 +deepspeed/inference/v2/modules/implementations/unembed/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/unembed/__pycache__/ragged_unembed.cpython-310.pyc,, +deepspeed/inference/v2/modules/implementations/unembed/ragged_unembed.py,sha256=cAd3By6SBykoCHJI3oxwE0FXHcMP5HEF5CSrWJyBjjg,4921 +deepspeed/inference/v2/modules/interfaces/__init__.py,sha256=2djwkw0m2KAAuBV-LuWY0W0ncwIrCxDX5uwf4oc8E-U,519 +deepspeed/inference/v2/modules/interfaces/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/modules/interfaces/__pycache__/attention_base.cpython-310.pyc,, +deepspeed/inference/v2/modules/interfaces/__pycache__/embedding_base.cpython-310.pyc,, +deepspeed/inference/v2/modules/interfaces/__pycache__/linear_base.cpython-310.pyc,, +deepspeed/inference/v2/modules/interfaces/__pycache__/moe_base.cpython-310.pyc,, +deepspeed/inference/v2/modules/interfaces/__pycache__/post_norm_base.cpython-310.pyc,, +deepspeed/inference/v2/modules/interfaces/__pycache__/pre_norm_base.cpython-310.pyc,, +deepspeed/inference/v2/modules/interfaces/__pycache__/unembed_base.cpython-310.pyc,, +deepspeed/inference/v2/modules/interfaces/attention_base.py,sha256=Kl8aPbpzyhP4bF21LZ_OcEwHFoI1KFirOiukiV4YajE,3613 +deepspeed/inference/v2/modules/interfaces/embedding_base.py,sha256=3pXB9F4IlWMqFGV8ZYE3k3bG2Je4B9uvVnVS0pRzLeU,3071 +deepspeed/inference/v2/modules/interfaces/linear_base.py,sha256=8g8or7xUMtGX_3uQj6zt0YfUBS9SwSbRY02lrr3gUVE,2146 +deepspeed/inference/v2/modules/interfaces/moe_base.py,sha256=5BCoKWZplUfxMj7mOPA58b4aBGk7APM-WG-ByxUIUyA,2850 +deepspeed/inference/v2/modules/interfaces/post_norm_base.py,sha256=PktjhFD5Pi3y2zt_VEwB9L-yNVyIo4dHpJgdVwgbcVc,2123 +deepspeed/inference/v2/modules/interfaces/pre_norm_base.py,sha256=QlYoS3N4N3ZKfUJk6C9u_vCT10p4-ACVgAEydLquHnY,2106 +deepspeed/inference/v2/modules/interfaces/unembed_base.py,sha256=W5ToOG46DZwH5e1VnFXxXp_f9-EvnO2iIw6zqZkL38w,2055 +deepspeed/inference/v2/modules/module_registry.py,sha256=UyDd-5FYQwbEJ7ihPZ60GkEiQ7M6Z2XMWnEAaLZN_WQ,2073 +deepspeed/inference/v2/ragged/__init__.py,sha256=2fFLZ35zp9WgcDaQ3XaTti8Hzi99zt8EgueCaaOqZs4,418 +deepspeed/inference/v2/ragged/__pycache__/__init__.cpython-310.pyc,, +deepspeed/inference/v2/ragged/__pycache__/blocked_allocator.cpython-310.pyc,, +deepspeed/inference/v2/ragged/__pycache__/kv_cache.cpython-310.pyc,, +deepspeed/inference/v2/ragged/__pycache__/manager_configs.cpython-310.pyc,, +deepspeed/inference/v2/ragged/__pycache__/ragged_manager.cpython-310.pyc,, +deepspeed/inference/v2/ragged/__pycache__/ragged_wrapper.cpython-310.pyc,, +deepspeed/inference/v2/ragged/__pycache__/sequence_descriptor.cpython-310.pyc,, +deepspeed/inference/v2/ragged/blocked_allocator.py,sha256=8rwqli63wrLAbzOuLoOgVqLi5UlpPlyxXtyRS1gU6zM,3661 +deepspeed/inference/v2/ragged/csrc/fast_host_buffer.cu,sha256=8U10U6VZk12oGwXCniQsqduta_cEcVBFuN9Fq3LO1JQ,501 +deepspeed/inference/v2/ragged/csrc/ragged_ops.cpp,sha256=Z6na7zRDPD7PrYbZ4lBhBsGyMatZ9yUNYw_PBzRGHw8,2726 +deepspeed/inference/v2/ragged/includes/fast_host_buffer.h,sha256=N73g7w1uE1gEsyWzrEnEibGeU8nBJPDeL-SWJEWK_0M,302 +deepspeed/inference/v2/ragged/kv_cache.py,sha256=eekwPz2OMNgQJHYXcxIrW86yK0guS419GeNqXJ38wIg,8559 +deepspeed/inference/v2/ragged/manager_configs.py,sha256=_SDSRv5VC5ZdmN6VDckjHtuC4HdJK6KjH9Vec839WcM,5965 +deepspeed/inference/v2/ragged/ragged_manager.py,sha256=Mk09dkqkEj5CMA9kWvTWuQeiw7fWiO9xn_nMIs-1Hho,7334 +deepspeed/inference/v2/ragged/ragged_wrapper.py,sha256=RJJpqSVz6vsElbtfePx99TDYZ2Q_Hao7qt_2Jt6RP2s,12701 +deepspeed/inference/v2/ragged/sequence_descriptor.py,sha256=oo1kf0upitTJkbZB6ME7GY3_MoOnGABXcw1coq49ljM,10891 +deepspeed/inference/v2/scheduling_utils.py,sha256=S1MPmx8zU8qHmDd_eDLhKP6hh71N9lo_Q8kJxO1AyOY,1363 +deepspeed/io/__init__.py,sha256=948yAWx0rWW6wzxU7f2JTmwF88rNisJdS1FoPs_JNZc,249 +deepspeed/io/__pycache__/__init__.cpython-310.pyc,, +deepspeed/io/__pycache__/base_file_writer.cpython-310.pyc,, +deepspeed/io/__pycache__/base_io_buffer.cpython-310.pyc,, +deepspeed/io/__pycache__/constants.cpython-310.pyc,, +deepspeed/io/__pycache__/double_io_buffer.cpython-310.pyc,, +deepspeed/io/__pycache__/fast_file_writer.cpython-310.pyc,, +deepspeed/io/__pycache__/mock_file_writer.cpython-310.pyc,, +deepspeed/io/__pycache__/py_file_writer.cpython-310.pyc,, +deepspeed/io/__pycache__/single_io_buffer.cpython-310.pyc,, +deepspeed/io/__pycache__/utils.cpython-310.pyc,, +deepspeed/io/base_file_writer.py,sha256=nXR4OuhLiHga6-Colhu578n97LP8sEkXCwKfFKOZ5rw,1007 +deepspeed/io/base_io_buffer.py,sha256=C-FVSF4w6wc7lCZ9VXkRhwBnW75TUE-n_tTko4YplnU,1841 +deepspeed/io/constants.py,sha256=9nxuv0_QNaB_1CN9d9FXvlEPrZ2fEHFL3yuo8eoshZg,806 +deepspeed/io/double_io_buffer.py,sha256=6qRY3Je1o2onzvXIP50KZdfJfzraYzBCBlRJoC-K0xM,3028 +deepspeed/io/fast_file_writer.py,sha256=lh8RGJc9JAKKNrzKkyOh28DCpLoh_DHqUatNbC-WJIU,11049 +deepspeed/io/mock_file_writer.py,sha256=pWXxx5o7DyV1xKsAOiWoSUH9ZIYDIlvRtlS8JHTkj-4,1638 +deepspeed/io/py_file_writer.py,sha256=knJ292prjk4AdfbbSiDEK0DGoBsZU6rMG_RYqo7KcAU,906 +deepspeed/io/single_io_buffer.py,sha256=CeUIj3GO-CnfTWHA9wrqe89W_PRjOF7GjXJYuCE36i4,1324 +deepspeed/io/utils.py,sha256=D0IOzZmhByrqOBQxP5m3KvA2HGanM3tV0OxZXQyMyas,1464 +deepspeed/launcher/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/launcher/__pycache__/__init__.cpython-310.pyc,, +deepspeed/launcher/__pycache__/constants.cpython-310.pyc,, +deepspeed/launcher/__pycache__/launch.cpython-310.pyc,, +deepspeed/launcher/__pycache__/launcher_helper.cpython-310.pyc,, +deepspeed/launcher/__pycache__/multinode_runner.cpython-310.pyc,, +deepspeed/launcher/__pycache__/runner.cpython-310.pyc,, +deepspeed/launcher/constants.py,sha256=X-6bI3aC4NXSZgrMJ6LaIILrsWu_TNteNBwgKOhNrgI,375 +deepspeed/launcher/launch.py,sha256=aHutEg7kj_kY5VdH_CQLclyYOIZoh2YSRLD-wXvM_BI,14898 +deepspeed/launcher/launcher_helper.py,sha256=tpvx6ItaLbBNYnUwP8XoKcn8P0mFXRffpB1Z_uLB_U0,3931 +deepspeed/launcher/multinode_runner.py,sha256=QcTvKOnkP94gmjLrK9om9Upw8FKClAmSLyCEtKQFKjU,18797 +deepspeed/launcher/runner.py,sha256=3u_dE4Q1e0bTmFjwk85JVRRWwbRmnzYmnloS0crJS_0,26306 +deepspeed/linear/__init__.py,sha256=fwQSoiw98NsTIwp6BuhtNMykJsaJOpJWd5QAUpitfcc,238 +deepspeed/linear/__pycache__/__init__.cpython-310.pyc,, +deepspeed/linear/__pycache__/config.cpython-310.pyc,, +deepspeed/linear/__pycache__/context_manager.cpython-310.pyc,, +deepspeed/linear/__pycache__/optimized_linear.cpython-310.pyc,, +deepspeed/linear/__pycache__/quantization.cpython-310.pyc,, +deepspeed/linear/config.py,sha256=U4yr-hEialc28KiEp4qRfm-8H_SF5xsLij2DROOTGSQ,2178 +deepspeed/linear/context_manager.py,sha256=L_aADkH1e59C7LCpoS7XyxC3rJ3QBSBqXucrVOjy1ss,3574 +deepspeed/linear/optimized_linear.py,sha256=RcpH8q8QpB7hE85DqU_evn29xVxKcWxOYzvx8aiUaUQ,10474 +deepspeed/linear/quantization.py,sha256=Lpvwk_-yb3EQpsNQC1sFT9fGvlj6dPdQx5fg0Rxb0s8,6352 +deepspeed/model_implementations/__init__.py,sha256=jJCJcnbvmFc9eQxIUIb7Gd6Ysd8ijSuuk-aaqFrDkK4,220 +deepspeed/model_implementations/__pycache__/__init__.cpython-310.pyc,, +deepspeed/model_implementations/diffusers/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/model_implementations/diffusers/__pycache__/__init__.cpython-310.pyc,, +deepspeed/model_implementations/diffusers/__pycache__/unet.cpython-310.pyc,, +deepspeed/model_implementations/diffusers/__pycache__/vae.cpython-310.pyc,, +deepspeed/model_implementations/diffusers/unet.py,sha256=W2_9-c-5Tz-nymu7kK2du2rzpV8-eC8JZEpQGRhGMek,3056 +deepspeed/model_implementations/diffusers/vae.py,sha256=xbjGaYsXeJA4nu49gQaGO5GrEazCHwcTUVqPsxoOTn4,6244 +deepspeed/model_implementations/features/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/model_implementations/features/__pycache__/__init__.cpython-310.pyc,, +deepspeed/model_implementations/features/__pycache__/cuda_graph.cpython-310.pyc,, +deepspeed/model_implementations/features/cuda_graph.py,sha256=-KgILcht5qw_ayzI0CqSPGGFoqZKHG8E5CTfD8ACgw8,563 +deepspeed/model_implementations/transformers/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/model_implementations/transformers/__pycache__/__init__.cpython-310.pyc,, +deepspeed/model_implementations/transformers/__pycache__/clip_encoder.cpython-310.pyc,, +deepspeed/model_implementations/transformers/__pycache__/ds_base.cpython-310.pyc,, +deepspeed/model_implementations/transformers/__pycache__/ds_bert.cpython-310.pyc,, +deepspeed/model_implementations/transformers/__pycache__/ds_bloom.cpython-310.pyc,, +deepspeed/model_implementations/transformers/__pycache__/ds_gpt.cpython-310.pyc,, +deepspeed/model_implementations/transformers/__pycache__/ds_llama2.cpython-310.pyc,, +deepspeed/model_implementations/transformers/__pycache__/ds_megatron_gpt.cpython-310.pyc,, +deepspeed/model_implementations/transformers/__pycache__/ds_opt.cpython-310.pyc,, +deepspeed/model_implementations/transformers/__pycache__/ds_transformer.cpython-310.pyc,, +deepspeed/model_implementations/transformers/clip_encoder.py,sha256=BTc_do6u4A79uhoNwQYxKrbqSnk2C63BO6xN0Ez-fz0,3096 +deepspeed/model_implementations/transformers/ds_base.py,sha256=feBwoCtVctmdYb_KZruLVCJr7WUBsfNZtVUxEKtO5MQ,388 +deepspeed/model_implementations/transformers/ds_bert.py,sha256=fckfXCn3zf9DsvqiyVAb0pwn25GA9H4ZceaCI4gLLSI,667 +deepspeed/model_implementations/transformers/ds_bloom.py,sha256=_YHlkBOUBTT27ZRWs7x6DxKz_5gILGBAe2YNmlkCij0,669 +deepspeed/model_implementations/transformers/ds_gpt.py,sha256=PkhQU0iHFkSGRxaEb_j1UDvRWruwrOPF8MLZfAo55ZY,665 +deepspeed/model_implementations/transformers/ds_llama2.py,sha256=geGAyCXgRNzMlk0xhvqu3FFmHfFgotVXT6aVV4ZlsYo,2128 +deepspeed/model_implementations/transformers/ds_megatron_gpt.py,sha256=kVtyp2Uss9Z_MRGwgAvJI_2tfWlPgzMpywIPS0xeOIo,682 +deepspeed/model_implementations/transformers/ds_opt.py,sha256=oHS210T27R5zOtg8ik_QwV22F8BIJxKVWPiPQaDXzjY,665 +deepspeed/model_implementations/transformers/ds_transformer.py,sha256=YjGCBKUjf_yZ1FJw2vEN-SR73dp4ndK1OknALNsvgKQ,8838 +deepspeed/module_inject/__init__.py,sha256=AechzL-WB6riK3mtss4gUD4HD79MhxZzwmPpZzBJSSk,461 +deepspeed/module_inject/__pycache__/__init__.cpython-310.pyc,, +deepspeed/module_inject/__pycache__/auto_tp.cpython-310.pyc,, +deepspeed/module_inject/__pycache__/auto_tp_model_utils.cpython-310.pyc,, +deepspeed/module_inject/__pycache__/fusedqkv_utils.cpython-310.pyc,, +deepspeed/module_inject/__pycache__/inject.cpython-310.pyc,, +deepspeed/module_inject/__pycache__/layers.cpython-310.pyc,, +deepspeed/module_inject/__pycache__/load_checkpoint.cpython-310.pyc,, +deepspeed/module_inject/__pycache__/module_quantize.cpython-310.pyc,, +deepspeed/module_inject/__pycache__/policy.cpython-310.pyc,, +deepspeed/module_inject/__pycache__/replace_module.cpython-310.pyc,, +deepspeed/module_inject/__pycache__/replace_policy.cpython-310.pyc,, +deepspeed/module_inject/__pycache__/tp_shard.cpython-310.pyc,, +deepspeed/module_inject/__pycache__/utils.cpython-310.pyc,, +deepspeed/module_inject/auto_tp.py,sha256=KAyjZgGCmaQ5_HRgUQvjBgUikmecpnXZw9q62eYQFyw,24809 +deepspeed/module_inject/auto_tp_model_utils.py,sha256=G_tPVAaT7b422Wa8J785rEOcM742uXnoW-5fvd1cinU,5910 +deepspeed/module_inject/containers/__init__.py,sha256=AQbuSXpxEyLpLN3RoAkQeyvtioX_8ShH6C8cqzcQ2Jw,1015 +deepspeed/module_inject/containers/__pycache__/__init__.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/base.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/base_moe.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/bert.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/bloom.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/clip.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/distil_bert.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/gpt2.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/gptj.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/gptneo.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/gptneox.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/internlm.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/llama.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/llama2.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/megatron_gpt.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/megatron_gpt_moe.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/opt.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/unet.cpython-310.pyc,, +deepspeed/module_inject/containers/__pycache__/vae.cpython-310.pyc,, +deepspeed/module_inject/containers/base.py,sha256=kw-culwCQ480zVCgeY7K9LH30BSuKWuPtm2lBMh-L_o,13762 +deepspeed/module_inject/containers/base_moe.py,sha256=lfyM4AhAXmYp5dxa60XoEbcJDQVj9uB-pRlzIDR2ypI,5756 +deepspeed/module_inject/containers/bert.py,sha256=f0VMB8R5mCHxNgIVhQSYeTgVsxo1IN4BV6vzSIOgLZc,3769 +deepspeed/module_inject/containers/bloom.py,sha256=fV_vVFyjB0BB95kP7TrjwqrqruKbEwWQH6ttduzfcwg,6256 +deepspeed/module_inject/containers/clip.py,sha256=KR3s7ev3v-ccvrFhqfm2pKjApQeYeCfhm4aZZ0_Ou_A,2822 +deepspeed/module_inject/containers/distil_bert.py,sha256=j8ufi0mXYe862fLYMtY1J0Rh2uY-xzkXKDKrhK-EsgI,3188 +deepspeed/module_inject/containers/features/__init__.py,sha256=GOivP3ix552VHgGPZ70UehyLQ_YZ8TLRc0-rgTCO07c,275 +deepspeed/module_inject/containers/features/__pycache__/__init__.cpython-310.pyc,, +deepspeed/module_inject/containers/features/__pycache__/gated_mlp.cpython-310.pyc,, +deepspeed/module_inject/containers/features/__pycache__/hybrid_engine.cpython-310.pyc,, +deepspeed/module_inject/containers/features/__pycache__/hybrid_megatron.cpython-310.pyc,, +deepspeed/module_inject/containers/features/__pycache__/megatron.cpython-310.pyc,, +deepspeed/module_inject/containers/features/__pycache__/meta_tensor.cpython-310.pyc,, +deepspeed/module_inject/containers/features/__pycache__/split_qkv.cpython-310.pyc,, +deepspeed/module_inject/containers/features/gated_mlp.py,sha256=ep8fnRs-06C7oL75w5UBAaLyCb9BwAx2v3c-sBhkB2k,5397 +deepspeed/module_inject/containers/features/hybrid_engine.py,sha256=ki_k_0SstM39XfN0SNWwiSO3qzIBs2qKxxAyrGxpfUo,8374 +deepspeed/module_inject/containers/features/hybrid_megatron.py,sha256=qp04fT0ManTJwzVdABFP6OWvLQsD6pWDaJAxNJU4-ZQ,4129 +deepspeed/module_inject/containers/features/megatron.py,sha256=BrbmtBC9ZKFL4ESbIYRJLzoEnEp3QOhkj1VcoqlcEek,1200 +deepspeed/module_inject/containers/features/meta_tensor.py,sha256=oem92mPilw6_UOBzkE-7VMoi9vpA9kvA1lemDcCSPwQ,2928 +deepspeed/module_inject/containers/features/split_qkv.py,sha256=totYVBg1M6DcsdC3762Vhoq7gpGJ1BIioTfDFdFwkF0,7136 +deepspeed/module_inject/containers/gpt2.py,sha256=98yUKFAGrfyyCv5yK7V4Zr8oWf6m4qTXgGIzUbRG0kM,2221 +deepspeed/module_inject/containers/gptj.py,sha256=OJ4Z-G5sjd-92yrNDJ0D53Jz_o0n2tfkg9fS9Mll1ss,5074 +deepspeed/module_inject/containers/gptneo.py,sha256=NakhEgR8qh-U9rZ6JD6nIyY_r1mH2obkrgLat4eKpMo,5790 +deepspeed/module_inject/containers/gptneox.py,sha256=zM8hYq0e1FKrDd0n0m1oUPKKMDNpwxVmXQHm8jlv1CA,5897 +deepspeed/module_inject/containers/internlm.py,sha256=3JYddGlL-87jC35MSKvYw9XzZFHYIIYC4Un0Ek5jsL4,7761 +deepspeed/module_inject/containers/llama.py,sha256=ziPp5QSofVH7nTVbsCN0ljYlwc9HZi5ZiS_YSFWi_HA,6926 +deepspeed/module_inject/containers/llama2.py,sha256=5WbtiVVAL1dVseHiO5kvx2qOwVAAvq-cKBKo0JZ7YjA,6359 +deepspeed/module_inject/containers/megatron_gpt.py,sha256=PiD58-Ul61ZJE4QAkCKCtGwjg44wMaSPOkqnWcRudyo,5417 +deepspeed/module_inject/containers/megatron_gpt_moe.py,sha256=xFOff3OOIKMz2YAFRrqLHxpkzLgB-chA4L69HXiey98,3936 +deepspeed/module_inject/containers/opt.py,sha256=exIuEYIu-KCjwPL3L-ykz1ud477q-F4WBwbXIL0Hef4,6905 +deepspeed/module_inject/containers/unet.py,sha256=fWteUHNx0S8u6C3xpiU2TLK4snlTr7Bv81yCNlVIDY8,1862 +deepspeed/module_inject/containers/vae.py,sha256=yLb5XWwZcaVOXVbUjGW1n0hUr3wrL4rqzqFkvBbD1Y4,1505 +deepspeed/module_inject/fusedqkv_utils.py,sha256=GvycFptz45D6k8eCPqJ-jJs4Vui8QzoyqK0CLVqbICs,10188 +deepspeed/module_inject/inject.py,sha256=KkpC_LLybQeiFzIhr6pz-OlKOl92XNFf3Vh7GqD7jsY,4719 +deepspeed/module_inject/layers.py,sha256=TcMmFqGcapbLQLiZUSlpBFuavpo3PB8V6TCbY7JZ5Bc,31475 +deepspeed/module_inject/load_checkpoint.py,sha256=BwyqWbY18JGZJv709sKG3_ZSUP1lGUjFnP8KYDpESgM,15434 +deepspeed/module_inject/module_quantize.py,sha256=NrOwvSfPWWOnlUVe82sSOZhay76JzqkgZ-j4NP-yEOY,3144 +deepspeed/module_inject/policy.py,sha256=kfyIzx4R4INZve94RPBPZu0jnnQKYpWtou-nRvZQ0Z0,8259 +deepspeed/module_inject/replace_module.py,sha256=_pfrNWVj286Qv4gQYglPWSiMvumKjBj2Tkam7CH4cL4,33068 +deepspeed/module_inject/replace_policy.py,sha256=jlIhG7pbXcjD8JPjWJCU-Gs_WSTG_q7K-IHmewu04pw,1119 +deepspeed/module_inject/tp_shard.py,sha256=n7n9vJYwPMuJVJjN5UaMiDwiPI-xTDOi8oaJMRNKOxg,2126 +deepspeed/module_inject/utils.py,sha256=q5gZWY7YK_-BI7ce16-G8dvU9Qgx7Qbezz9Hwz7oKS0,1995 +deepspeed/moe/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/moe/__pycache__/__init__.cpython-310.pyc,, +deepspeed/moe/__pycache__/experts.cpython-310.pyc,, +deepspeed/moe/__pycache__/layer.cpython-310.pyc,, +deepspeed/moe/__pycache__/mappings.cpython-310.pyc,, +deepspeed/moe/__pycache__/sharded_moe.cpython-310.pyc,, +deepspeed/moe/__pycache__/utils.cpython-310.pyc,, +deepspeed/moe/experts.py,sha256=7xFMeuKcBmfyVdaVKpUw5uyGzHYbtxXTiKqcUuBjrQ8,1315 +deepspeed/moe/layer.py,sha256=TayzIRGqEOakrq4TthCiBpqdwwdm5sGqISWMZG13KMs,6905 +deepspeed/moe/mappings.py,sha256=S-o5bwyGoZPmKL0i2E2TrWakyri_P7zARLgOY_4a8u8,4090 +deepspeed/moe/sharded_moe.py,sha256=7z2v3inspO0O8YSGWTll036Xxc3_TiB-vECG2wlKy9E,27332 +deepspeed/moe/utils.py,sha256=YGnqMJD26e2q3ezp-A0srHyb2sdWaHUEg1ld_I6qPgU,6699 +deepspeed/monitor/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/monitor/__pycache__/__init__.cpython-310.pyc,, +deepspeed/monitor/__pycache__/comet.cpython-310.pyc,, +deepspeed/monitor/__pycache__/config.cpython-310.pyc,, +deepspeed/monitor/__pycache__/csv_monitor.cpython-310.pyc,, +deepspeed/monitor/__pycache__/monitor.cpython-310.pyc,, +deepspeed/monitor/__pycache__/tensorboard.cpython-310.pyc,, +deepspeed/monitor/__pycache__/utils.cpython-310.pyc,, +deepspeed/monitor/__pycache__/wandb.cpython-310.pyc,, +deepspeed/monitor/comet.py,sha256=16xIXzwylfuGJyt4p3Lu46JXemCe89_2XCi_cLoWDlY,2903 +deepspeed/monitor/config.py,sha256=vdDaZ7qM0RLXIStjW6S2C4vJimCx0g0IQIi6Hv_sn4k,5258 +deepspeed/monitor/csv_monitor.py,sha256=E9bWWx7QKhex8GWOxMOLjOsrAyBmXqCkaxNUyVirSY0,2907 +deepspeed/monitor/monitor.py,sha256=d_Ke7YvSsXpXeyLmx61KaqkYpgG026sSSbMel0-lfVg,1894 +deepspeed/monitor/tensorboard.py,sha256=9crE_YD9adHS65QmRxAuSuezRQUpL_VHrJMZmPhmuiE,2227 +deepspeed/monitor/utils.py,sha256=MCyfYHyZL-F1a4jbEqG5_d2-1CvX81-gZC6JPlkIblY,1226 +deepspeed/monitor/wandb.py,sha256=G-2BnrWGjiuWG9Tg5-bzkBsjRLwlsyINAYryxOmMdvE,1128 +deepspeed/nebula/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/nebula/__pycache__/__init__.cpython-310.pyc,, +deepspeed/nebula/__pycache__/config.cpython-310.pyc,, +deepspeed/nebula/__pycache__/constants.cpython-310.pyc,, +deepspeed/nebula/config.py,sha256=THO1Mwto2utxnBWRfn0vIVWevOpT2EPFJFZkDCJF-ck,1764 +deepspeed/nebula/constants.py,sha256=0HQOkViV_lRv13vtczIhBJG_YAH876vhh0R4BqHAj68,2786 +deepspeed/nvme/__init__.py,sha256=iPQdRvD-GrL7bbRWcJtX0TFD7kz-OZVTmHDQa1mG3Z0,241 +deepspeed/nvme/__pycache__/__init__.cpython-310.pyc,, +deepspeed/nvme/__pycache__/ds_aio_args.cpython-310.pyc,, +deepspeed/nvme/__pycache__/ds_aio_basic.cpython-310.pyc,, +deepspeed/nvme/__pycache__/ds_aio_constants.cpython-310.pyc,, +deepspeed/nvme/__pycache__/ds_aio_handle.cpython-310.pyc,, +deepspeed/nvme/__pycache__/ds_aio_job.cpython-310.pyc,, +deepspeed/nvme/__pycache__/io_engine.cpython-310.pyc,, +deepspeed/nvme/__pycache__/parse_nvme_stats.cpython-310.pyc,, +deepspeed/nvme/__pycache__/perf_generate_param.cpython-310.pyc,, +deepspeed/nvme/__pycache__/perf_run_sweep.cpython-310.pyc,, +deepspeed/nvme/__pycache__/perf_sweep_utils.cpython-310.pyc,, +deepspeed/nvme/__pycache__/test_ds_aio.cpython-310.pyc,, +deepspeed/nvme/__pycache__/test_ds_aio_utils.cpython-310.pyc,, +deepspeed/nvme/__pycache__/torch_fastio_engine.cpython-310.pyc,, +deepspeed/nvme/__pycache__/torch_io.cpython-310.pyc,, +deepspeed/nvme/__pycache__/validate_async_io.cpython-310.pyc,, +deepspeed/nvme/ds_aio_args.py,sha256=gNjCXHv3VyyioJAWrMoAx4HJYmrN68VNNdq9pwly7SE,8458 +deepspeed/nvme/ds_aio_basic.py,sha256=WXImsGkd3E5qwuMOjZ6wHhCHQM7tzt7nKyg-LzeUpVg,2410 +deepspeed/nvme/ds_aio_constants.py,sha256=Af0dLTpRrw8VXmMby28zy60_tEWx61i15mpUSNzk51E,493 +deepspeed/nvme/ds_aio_handle.py,sha256=HS-sNFfUgwJn6T8o0pZjdK2gMQt4_xLzzHXu-M7PXtU,4944 +deepspeed/nvme/ds_aio_job.py,sha256=v-4U2oTjTqPZ7jjg1yIDtZOMlwfOQcOPMKCmzqTtAQQ,1276 +deepspeed/nvme/io_engine.py,sha256=nUJ1cXfLg6x3IQQ9g-zILY2I584beeigBaTHbsn2lPs,3847 +deepspeed/nvme/parse_nvme_stats.py,sha256=UjaSbCHcP0MifIngLT7mmWbf1BbkKRb-VYO8WbfBmqc,3835 +deepspeed/nvme/perf_generate_param.py,sha256=VPIleO_Ihe5JJRARe94DrXf_sWNCgC8RBPIwz7H_tYI,3289 +deepspeed/nvme/perf_run_sweep.py,sha256=PaLX4sfWHjbIgIJiG3W3t_Mgll_NfHCm6Qmp627Kwb4,9747 +deepspeed/nvme/perf_sweep_utils.py,sha256=pvreOkVAjNHwxSQ33bgXr4Yelwjzy-l9SEOcFTKGjhw,425 +deepspeed/nvme/test_ds_aio.py,sha256=0v_ZTRfro2gjANPa9CVoii1gz021Z5Eol1z9_hbLlR8,590 +deepspeed/nvme/test_ds_aio_utils.py,sha256=BORiau54UKygrVdLsqQLIq1e90ED3PsW1X6WI9DO2s8,3103 +deepspeed/nvme/torch_fastio_engine.py,sha256=zf4S1zsoE64jHUx9hrwYY-Ciy_6GSUC1O0x9rsL5DnQ,3509 +deepspeed/nvme/torch_io.py,sha256=gIq32rm0kRVYP_R-RFZphkmVVDv3TnJVYD8JjKx2gWg,2270 +deepspeed/nvme/validate_async_io.py,sha256=RsuUU_w27HuHWZAKsan5M0-pcllXRyf_H_VQcs-U4T8,302 +deepspeed/ops/__init__.py,sha256=jr72zRfZAdER7PTHnZc-9GziYpnxW80sBCyBthYWYhI,407 +deepspeed/ops/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/adagrad/__init__.py,sha256=aJHYZSC21yulH09u72ZYAe43nLgeMemqvmC9wNNh9CQ,141 +deepspeed/ops/adagrad/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/adagrad/__pycache__/cpu_adagrad.cpython-310.pyc,, +deepspeed/ops/adagrad/cpu_adagrad.py,sha256=nXZLRyBmJS4hcdt6XULXVrX7hQq09nrLFFajJTk5do8,4181 +deepspeed/ops/adam/__init__.py,sha256=ZVagnjkzHVw4akcDibUCfsArTcu8J9wsz9xKmbrJnIs,169 +deepspeed/ops/adam/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/adam/__pycache__/cpu_adam.cpython-310.pyc,, +deepspeed/ops/adam/__pycache__/fused_adam.cpython-310.pyc,, +deepspeed/ops/adam/__pycache__/multi_tensor_apply.cpython-310.pyc,, +deepspeed/ops/adam/cpu_adam.py,sha256=URu3D8FceMISyD_-49MgPYWRHJ2SS59deX7sGqnQAKQ,7520 +deepspeed/ops/adam/fused_adam.py,sha256=5_JBk_UvBFcnOL0_hPr5SVFATmmmKHFIlVSR1am-sBc,8767 +deepspeed/ops/adam/multi_tensor_apply.py,sha256=APt3UCnfw-nLjJOGWXC3izWRXNu-16TYhB-s213DpM0,429 +deepspeed/ops/aio/__init__.py,sha256=dHc8QXzdN4Cw0D7px-gbO_fzV-lKT21ySZJ67LtUJuI,136 +deepspeed/ops/aio/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/compile/__init__.py,sha256=6GIOlNEFXSRcqWtTL1JcczbQQcLh2Ab1oBVcnlnii0I,140 +deepspeed/ops/compile/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/csrc/adagrad/cpu_adagrad.cpp,sha256=FPSi-AtroLc-zPnzN1yF2OnKkb-wCpKALqN_e8wKTPI,7455 +deepspeed/ops/csrc/adam/cpu_adam.cpp,sha256=9bS2_kdvipPCUoonzJNR6CLuCjNviTPcZJlzaNqGxGY,409 +deepspeed/ops/csrc/adam/cpu_adam_impl.cpp,sha256=QycbyNXh4VGNq4FihHwBysu-_oHSWf1d-STXBgtKZFE,8468 +deepspeed/ops/csrc/adam/fused_adam_frontend.cpp,sha256=tu6pBcAvqcdh1ti6SB7fj8HnqFDQg3Rqbaa4xsQ2v7M,880 +deepspeed/ops/csrc/adam/multi_tensor_adam.cu,sha256=13ixr7zAf-vH8NzAPKYiKXPmVRgsJxILT6cj75_92j8,8417 +deepspeed/ops/csrc/adam/multi_tensor_apply.cuh,sha256=tcBmATZa_Et2VH9hnq5Jf6LJNnLaWDNrSqHdRr2yCE4,5609 +deepspeed/ops/csrc/aio/common/deepspeed_aio_common.cpp,sha256=hZ8EYNZ-Lcrll0VVnvmb-Gd7mr2CZ_P2a0F9_9bnexo,13274 +deepspeed/ops/csrc/aio/common/deepspeed_aio_common.h,sha256=8fp1mBNfXIAgE48vL5CEtH_n6PeP8TeVs3UXkm-4lXs,1358 +deepspeed/ops/csrc/aio/common/deepspeed_aio_types.cpp,sha256=vrR8UQ9EK-8EgHBUyQIMgPLxNSFtwNUQVeoClggfPnQ,2033 +deepspeed/ops/csrc/aio/common/deepspeed_aio_types.h,sha256=pnFoIKThq3P_bHoo-jxZgL86SMbXlJZGfpghK7pQqBg,1402 +deepspeed/ops/csrc/aio/common/deepspeed_aio_utils.cpp,sha256=UihEk2VXEqQLCiT0u2ovXoQULBMGbZgSfNBwIx-csSQ,4674 +deepspeed/ops/csrc/aio/common/deepspeed_aio_utils.h,sha256=iCd_aS9H_DqJae7-O5qS9GASyE0gNYLLwwrJT-td_GY,2176 +deepspeed/ops/csrc/aio/py_lib/deepspeed_aio_op_desc.cpp,sha256=G23cmsy1S5u6_vOynCYJnO3Q1B4iZkvwpMEzcprcqQ8,1215 +deepspeed/ops/csrc/aio/py_lib/deepspeed_aio_op_desc.h,sha256=1n67p-zqla1BFNUPQ54Zrpq9OuWaKc7DuAUPKPDjScU,1065 +deepspeed/ops/csrc/aio/py_lib/deepspeed_aio_thread.cpp,sha256=pUuqwGkCkFLDtXtU5mDbgSTW7BXoFlAzFsTyIcx88GI,1394 +deepspeed/ops/csrc/aio/py_lib/deepspeed_aio_thread.h,sha256=fAX8as116Ecs7NObKX_LdjALHck0godLITqVkg-Byt4,885 +deepspeed/ops/csrc/aio/py_lib/deepspeed_cpu_op.cpp,sha256=Wye-QMpyW8AxLqqouStbdUDRzikV20ajizyskDygd0g,3433 +deepspeed/ops/csrc/aio/py_lib/deepspeed_cpu_op.h,sha256=lXYUW0DsMKdighnXSdKSMfmM4oZxpXCMJ71-Swtmf0c,1038 +deepspeed/ops/csrc/aio/py_lib/deepspeed_pin_tensor.cpp,sha256=MpYmRFffPgfkcjI_4tjw-nnEFimy1wgUxFF-HCNl5K4,1850 +deepspeed/ops/csrc/aio/py_lib/deepspeed_pin_tensor.h,sha256=oIJ0bbxcgE3XFwEMxSOx15XXk1rNbQFUNcYdHqFkQdU,861 +deepspeed/ops/csrc/aio/py_lib/deepspeed_py_aio.cpp,sha256=aZysoGV-1JU2wfAKQ3qYpyTGaAFPC8NDJGLQuucwRA0,4232 +deepspeed/ops/csrc/aio/py_lib/deepspeed_py_aio.h,sha256=z3O_AoPJXMHKjkwsrfrXIcIONXUNc2Yr3Pz1w3Q-99U,965 +deepspeed/ops/csrc/aio/py_lib/deepspeed_py_aio_handle.cpp,sha256=YvU1K6pKxcMG9knW0BRkbUh3jr-_N4ySp_QhDjIyudY,903 +deepspeed/ops/csrc/aio/py_lib/deepspeed_py_aio_handle.h,sha256=WgLYd05nnvNWO492PGBPNoIP8B0X2JI3axKFFSGLntQ,624 +deepspeed/ops/csrc/aio/py_lib/deepspeed_py_copy.cpp,sha256=xIqqhPWz-bqBD89HV9TItBnacCWr-JalPjqSaUodwrg,4414 +deepspeed/ops/csrc/aio/py_lib/deepspeed_py_copy.h,sha256=LxCbVzeTiKRmU2P_cbRZFHIInlW6HQWszpB5s_p2l0M,1169 +deepspeed/ops/csrc/aio/py_lib/deepspeed_py_io_handle.cpp,sha256=Frut6FvfUuJDMIPioBIJzXuuTR54Xjj6nPdBXNAclOo,12834 +deepspeed/ops/csrc/aio/py_lib/deepspeed_py_io_handle.h,sha256=1bDbRqqOuAOHlIO91k3MytydEQM4YLArmaCakBu5tig,3975 +deepspeed/ops/csrc/aio/py_lib/py_ds_aio.cpp,sha256=KGjMfQDWaKmViwB6cmJ7uZCJBK5HANTOkufHehFwz3I,4772 +deepspeed/ops/csrc/aio/py_test/single_process_config.json,sha256=6dSFspQHi0nNbLClaIfQ_3Jzj4Nw7urGCly8ZEWnfIo,433 +deepspeed/ops/csrc/compile/deepcompile.cpp,sha256=lar-g-tVzTCXzkrUXy3bevvZ4H65Se3hIfiQF5MUeHU,5868 +deepspeed/ops/csrc/compile/init.cpp,sha256=YfdOvY9YZGyWlo1JohwrsJe5RZuF6EibZekMyphzsGI,4461 +deepspeed/ops/csrc/compile/util.cpp,sha256=v9lmBKZDKPWRWOyi7Azve5PYGa5QO89uLwBHqlwzCCI,3769 +deepspeed/ops/csrc/compile/z1.cpp,sha256=rUEojV-gw-92ygEdS3hDBK52vE1cOfEU9nfz9jqXpMA,4391 +deepspeed/ops/csrc/compile/z1.h,sha256=gn4YNfbL21uG7McpLUhbmml8R7BRLJL9FTpUiwUyZoQ,459 +deepspeed/ops/csrc/compile/z2.cpp,sha256=ACKjRQS_TzisIyohi2HXVbRi_oL5fEsWd8boNO3b7fk,3973 +deepspeed/ops/csrc/compile/z2.h,sha256=HsOGpkbegv3bhZ65vKreTMAD8G2ZS1WJ0ZT-Vx5z_GM,247 +deepspeed/ops/csrc/compile/z3.cpp,sha256=pVo-ktWIbEYygvphK7E8CpBP3H6sV-_rUgy2ftrgcTM,20140 +deepspeed/ops/csrc/compile/z3.h,sha256=eRBbP0wfO15pLQtk4abWFNluBOHNGmOXEGiPQaj1WbY,2347 +deepspeed/ops/csrc/cpu/adam/fused_adam.cpp,sha256=m45BQnPCTkrYHiYcdKNa791zqVqtbdwia4TFoXjnOK0,1433 +deepspeed/ops/csrc/cpu/comm/ccl.cpp,sha256=1U2qIchXfW7ypHD1DIG7mjPXR2levhB55Jz00Kv5Hnk,12342 +deepspeed/ops/csrc/cpu/comm/shm.cpp,sha256=bcPXkJacINgDy081QimwLTBnXXxeUexV3zb_oxStsmk,27403 +deepspeed/ops/csrc/cpu/comm/shm.h,sha256=iM952l2aMWnNP7uI44blK4b8sS80Am756zWpjhd7K3o,401 +deepspeed/ops/csrc/cpu/comm/shm_interface.cpp,sha256=C19jMwHp4HhTinkHeFMdV7NtdMrq2JwUsXIcQK8CwnI,6195 +deepspeed/ops/csrc/cpu/lion/fused_lion.cpp,sha256=-_QUZhwjkjsePW-YyZ4ishpUPCGmbgfDp4C2jNgRIA0,1230 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/attention.cpp,sha256=1d5msQb6IiUdDeAYpzsiKjFHEFPHNAvBXDkO-aY2W_g,2095 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/attention_back.cu,sha256=10fKkFtrljPwiHjIAhoZJrBEcIRjdBP8JDagnFdWyOU,9971 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/attention_cu.cu,sha256=lfsGocib3dneilmyTKnEL9fdmALP8v_UhvwApmwgpVM,7509 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/epilogue/epilogue_grad_bias.h,sha256=ziZZar1z41wsr6ia5HtX8aAjpAuomtq5cCgZMzYylWI,12185 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/epilogue/epilogue_pipelined.h,sha256=RvYmxAW4NN7kMc9pLfEmkXRO-nXMKR1BX3fhhRUG3SI,24898 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/epilogue/epilogue_rescale_output.h,sha256=WWnsDQeR-XHcjKeEZixxhaTkv89tpyc1jaXoSAEiWt8,9606 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/epilogue/epilogue_thread_apply_logsumexp.h,sha256=Q_QP5LK3AwDtt6GHxLdUg-ttVqtDkxN2gv4bXiDnlPw,6392 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/gemm/custom_mma.h,sha256=rXA6J4g_TuQ1Ufm2c0Q2dMcuC1FV6X4MpkYpk7TeYp0,6270 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/gemm/custom_mma_base.h,sha256=zn3uylR7vb9GNLKG3UjodjXyPO_XpIuK0bRBwILm4zo,6511 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/gemm/custom_mma_multistage.h,sha256=WeVgJF-IIsVjd50TwDsL_oKwr2UMYUWdLEdhOiy_bfE,30281 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/gemm/custom_mma_pipelined.h,sha256=7MyvAD-eXwtNkfyjtd459IvbAbhW81Ys1RzFN8TasTM,15770 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/gemm/find_default_mma.h,sha256=l0wa_nFB9Vrz11bR_0lH-D4vaK-sWfuHm31Mg_Em3bc,9420 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/gemm/mma_accum_lambda_iterator.h,sha256=G6LeFpLrmBPVVsnfiYCPjh0h4VdJ5su6jyYPLvPymqI,16280 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/gemm/mma_from_smem.h,sha256=o_6L3rJlRtGmughlt9g0xLCzvTqBbUObl72lNxkbhNY,84216 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/gemm_kernel_utils.h,sha256=G02ENUZevpYkNI8FwYfxdiaY8BmFsZkIZh3ieGGWpzg,11661 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/iterators/epilogue_predicated_tile_iterator.h,sha256=Vl83Qgukp3lN0EwAkZdftc1gxIfvwvFzjRw7M8wSQ9A,27084 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/iterators/make_residual_last.h,sha256=bl7iRKlZYnjqYwoST7SRHWwqygjsvYCUmLMBfbAxV7Q,4245 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/iterators/predicated_tile_access_iterator_residual_last.h,sha256=Jg4qjZ0ukBy2ZIZ-2wxZ9u4mrIcdVAcil56e7CdCY8M,71866 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/iterators/predicated_tile_iterator_atomic.h,sha256=RQ1sW5cWwiPveRxIs8rVuOgaXBBhsgHU5mbiLh6rawc,31326 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/iterators/predicated_tile_iterator_residual_last.h,sha256=aZy_jvL9zakb9oeFXkoihOabhLLhZjRSbLkf_6lUwS8,71626 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/iterators/transpose_warp_iterator.h,sha256=PtMp-E5HhdBmwgRiEbE2ZjlVL-b97euA80waCNO7jYQ,2536 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/iterators/warp_iterator_from_smem.h,sha256=QvDXwcGimdRqSoaGSzN5aS3q37VSXeihlU_A6ZPee9M,10125 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/kernel_backward.h,sha256=lLlud-SKkjuOa9UcJkOUfSZRV3RQpPRf9J6qMtAODBQ,95042 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/kernel_forward.h,sha256=CTv6YV9cCkmzU7w49AqRhENGWOccHYz0d4YzmRhvVBM,47657 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/transform/bias_broadcast.h,sha256=7yLAnvgViEcY5qls5eEJyfQDNkToG0TfPKQhm8Qi0-E,5952 +deepspeed/ops/csrc/deepspeed4science/evoformer_attn/transform/tile_smem_loader.h,sha256=Dr_uSpu3dH8GAfChWggDeon9rn7F4G8BFrjm5_Xyt0E,4476 +deepspeed/ops/csrc/fp_quantizer/fp_quantize.cpp,sha256=74zRV9Yj3aoyA6rYhQmZUu0tVa4TqC7g3uR8mbytux0,5789 +deepspeed/ops/csrc/fp_quantizer/fp_quantize.cu,sha256=j17WEtr-J7xnSmOd4yt04OqKZawqPK-P7Aqdfk_omIM,25629 +deepspeed/ops/csrc/fp_quantizer/includes/context.h,sha256=n-WCCbWIdbfNBCeS8c1wdY8LNdGIkz8qwn8C9ImX_nM,1427 +deepspeed/ops/csrc/fp_quantizer/includes/fp_quantize.h,sha256=UTTOIZa18Cz0wOip7Tfn6itcCeUWySA0DJLDRZLOUY8,5979 +deepspeed/ops/csrc/gds/py_lib/deepspeed_gds_op.cpp,sha256=bmF57wG0eiTrKQ1WVR3eavTPfVPBSnxnb1i9tcUKQns,5815 +deepspeed/ops/csrc/gds/py_lib/deepspeed_gds_op.h,sha256=pD5329Gd2a75fzfThYDYHIb1Fs6TAeT0Br-YPjunHGg,1131 +deepspeed/ops/csrc/gds/py_lib/deepspeed_gds_utils.h,sha256=4abARh9qrz9TE9GkOUJOxZGzavRhQCubOFRuj-G3uLs,4790 +deepspeed/ops/csrc/gds/py_lib/deepspeed_py_gds_handle.cpp,sha256=X337agbrlRfSF-jnSzCLv4z_CejhaUyjyZowl8EoC4Q,4309 +deepspeed/ops/csrc/gds/py_lib/deepspeed_py_gds_handle.h,sha256=ZookPemckgcq6mTCuStAFxq47xmaRLnKGKeV7ca3BBA,1657 +deepspeed/ops/csrc/gds/py_lib/py_ds_gds.cpp,sha256=oJb2OJ-RoTtlqHc04Yz6DCg_rG9XZsayURECIbXaqwM,5106 +deepspeed/ops/csrc/includes/StopWatch.h,sha256=DocXNohnOPWiZBqfrdI6AKV9IggYRB1Bylzxfke_i5s,1981 +deepspeed/ops/csrc/includes/Timer.h,sha256=4mYxcATDJElSy5GCj_xFDZ2AvHHmMrhJlW0jJbMP7nQ,1180 +deepspeed/ops/csrc/includes/activation_type.h,sha256=WkFfkL5mgLyYAZimLUouThCkyA2aiF9LQlh1q206L9o,264 +deepspeed/ops/csrc/includes/compat.h,sha256=5ivuTerIqiCaYu2HpRPzJcUtqMsAtRB7aOXel69HeZQ,336 +deepspeed/ops/csrc/includes/context.h,sha256=Cs6hRXBiFSjpKKQOAMCX9F0uP_Hh1IeZLloNJxHxk1E,7207 +deepspeed/ops/csrc/includes/conversion_utils.h,sha256=augq8Zf-t46MaxWh0ryFb46_Guxud0k4kIK-h93rI8Y,12379 +deepspeed/ops/csrc/includes/cpu_adagrad.h,sha256=zcXXyFLZyjNSFfiXgaoYS8ytfFtvKyKQk8hP-SrmLuY,4036 +deepspeed/ops/csrc/includes/cpu_adam.h,sha256=Xkulb32K2B35yGaYqKKeBWEOBrhGb2160UqO-D2C0ok,7365 +deepspeed/ops/csrc/includes/cpu_lion.h,sha256=mFmdjrrWqll4mKKsNGeJysPomGFKBD1e5F2zosWSAq8,5516 +deepspeed/ops/csrc/includes/cublas_wrappers.h,sha256=Gyu-fwvPndgvI1PqGjb-CaoAkeGW8YsRRBBpIjKUjLg,3840 +deepspeed/ops/csrc/includes/custom_cuda_layers.h,sha256=lqmn3AfbfSynF2BLu3YWcZZnwQxsO89LiOluP-J2efU,12964 +deepspeed/ops/csrc/includes/deepcompile.h,sha256=qR0lZVVJCLzBxU_am8OxEOQIg8s4QB7yyRMragxyFLE,21211 +deepspeed/ops/csrc/includes/dequantization_utils.h,sha256=uFpPmBIYd3gNkKXImQLidYnGDYYTXkpY9V4o-AkdYpk,7301 +deepspeed/ops/csrc/includes/dropout.h,sha256=l5JuSzyfxRsfufBKIebPwpghnmYR_E6ucCmGbLvZ1qc,2195 +deepspeed/ops/csrc/includes/ds_kernel_utils.h,sha256=OKOMT46XItRnkrzqKHHK9n_ewxHh6AN-Uhv4aKixBm4,1297 +deepspeed/ops/csrc/includes/ds_transformer_cuda.h,sha256=OwVCwN4tS1kMEeF9J6QV9GG8AvtYklaHRJrMPmji-zY,6164 +deepspeed/ops/csrc/includes/feed_forward.h,sha256=8IbZYyUWVFGMhkjve5ZdWGQc8WvYXVhNEAddT2Vvp6s,3560 +deepspeed/ops/csrc/includes/gelu.h,sha256=iCsuAdNFzhLeccvc5D7KnsAcOcvXN_QD4jlaFRPF340,1018 +deepspeed/ops/csrc/includes/gemm_test.h,sha256=X9oIVKaZpFOGU0yCw-kUz4m5XDq3V1leW5v5dMGQYnA,11546 +deepspeed/ops/csrc/includes/general_kernels.h,sha256=0ksHr78VwjPiAfYPAloHM5uMxriRfNPEZyk9OklLhhI,1507 +deepspeed/ops/csrc/includes/memory_access_utils.h,sha256=Lhpfu-OXn3TscCXK6VdWr8SEp-9rFrtnThrQGDU6aQI,34901 +deepspeed/ops/csrc/includes/normalize_layer.h,sha256=j2lJkrxENFILqdUpEZIMj1qGrFty5SNoAeUrifThvSI,7089 +deepspeed/ops/csrc/includes/quantization.h,sha256=vqhmpPNw0Um-TqEtR_KyO3Pu6eZV5lGpXbraNLIfEoQ,5889 +deepspeed/ops/csrc/includes/quantization_utils.h,sha256=ciuykcFNUW9BxCYTE_moCtN5gJYFd6dgibCleBX7NdM,17366 +deepspeed/ops/csrc/includes/quantizer.h,sha256=YIRNwwr1eZYMmXbs6AZT82shpXuWbn94P4OVMcA3tss,346 +deepspeed/ops/csrc/includes/reduction_utils.h,sha256=AeQx6X3WD4vExcIQRO9LuHdCAWa5vLBcf2QmcIDgePM,23380 +deepspeed/ops/csrc/includes/simd.h,sha256=Ti4NkJ6ciwgPqEuAYmIr0zGRNPRq0z8gK8YmsRSZDKE,10190 +deepspeed/ops/csrc/includes/softmax.h,sha256=UGZHziAfbGN6OdTIA24G5CWHhV8hAfQbq16NN2_0Wlo,1642 +deepspeed/ops/csrc/includes/strided_batch_gemm.h,sha256=vp3ZGYjoSj_xZFTvdIj6PMrn_mupI3YlA4rLFjhYE8I,7279 +deepspeed/ops/csrc/includes/type_shim.h,sha256=8VuX9zK5EsmleO7wuAq9D1eHZC7zKWaf0N97eaGg7x8,6388 +deepspeed/ops/csrc/lamb/fused_lamb_cuda.cpp,sha256=icBfzI6QpN8LqIZT4-AY91UXlg_zDqkptNZy10erbws,3995 +deepspeed/ops/csrc/lamb/fused_lamb_cuda_kernel.cu,sha256=vjhQw1pj6v19tVE2otOCSStk53Ss0R82g9o1lXfJ5vw,15291 +deepspeed/ops/csrc/lion/cpu_lion.cpp,sha256=didiEmqBezyZ5C2yv92MVbPne8WQbDgkGN5JYV-e1ww,409 +deepspeed/ops/csrc/lion/cpu_lion_impl.cpp,sha256=xCsrkojmjl4uk-QSNa5TpONYRbAQi_V68yacdwvL5vs,7402 +deepspeed/ops/csrc/lion/fused_lion_frontend.cpp,sha256=RMiEOozd9gjXPidc4Ykg-kDIJek0OOASFVwmLIfFs0M,732 +deepspeed/ops/csrc/lion/multi_tensor_apply.cuh,sha256=tMXW3UwsOhNM4J1axQbNkn4gNfmig7_vXPiBLdFCCv4,5595 +deepspeed/ops/csrc/lion/multi_tensor_lion.cu,sha256=11RptKP9XmwMuvGBrmq3P1T8PAH9IvBJirB8gInMVoQ,4377 +deepspeed/ops/csrc/quantization/dequantize.cu,sha256=iibytiJox1QcqE8b57KCLPOaD610STv2VUmKoC7RVM8,3302 +deepspeed/ops/csrc/quantization/fake_quantizer.cu,sha256=CPMZnjOjJTKFibCBeYG1WrXnwEG-0l6POuZCRsBrHBo,37581 +deepspeed/ops/csrc/quantization/pt_binding.cpp,sha256=rcfjK9PldJDktjppS0_Rc9vFh_Rd_YSauGoyj6KzzAY,17434 +deepspeed/ops/csrc/quantization/quant_reduce.cu,sha256=mQ4rRfiPRDfgGJHgt5uFlgsBMav94aZqDYeui9JLBLs,25376 +deepspeed/ops/csrc/quantization/quantize.cu,sha256=IzmI5sq3V8B3tF92i84wlouV186djjVvdISREcepYRc,6988 +deepspeed/ops/csrc/quantization/quantize_intX.cu,sha256=SIEWB2qHtFkHINoTTNukiJnXnQS06BKVH3Wjtg-im1k,8514 +deepspeed/ops/csrc/quantization/swizzled_quantize.cu,sha256=dbNOMXtGQwBI7GnGZXKfVvR9h0iRC-aDU6jSeKrzoMo,19787 +deepspeed/ops/csrc/random_ltd/gather_scatter.cu,sha256=YFiLiYlgkq1RNYS8yRNI3CKYUrv_8ht0lsvZsUcsUYg,8408 +deepspeed/ops/csrc/random_ltd/pt_binding.cpp,sha256=7NAanMKS-j_hBputc6Ltv1yH6-c7xLwhDZJUfLiH--M,9590 +deepspeed/ops/csrc/random_ltd/slice_attn_masks.cu,sha256=KqANgsPTAAHyys0cPh3023JaBF8b0kOKF-W13W39Q_4,5076 +deepspeed/ops/csrc/random_ltd/token_sort.cu,sha256=Tm3oqRHM6UF1PWMuVaghnV0NftUjv3SUEbtBejNLJNc,7044 +deepspeed/ops/csrc/sparse_attention/utils.cpp,sha256=HmNSJfE5WzsE3dYSWjOKOAxawUOcgZ2JwQpJSjvB3b4,4523 +deepspeed/ops/csrc/spatial/csrc/opt_bias_add.cu,sha256=8N-FAKh-6Fwu7byz6e_sjXpd8XCRwdFntbZzZL6H8x8,6273 +deepspeed/ops/csrc/spatial/csrc/pt_binding.cpp,sha256=K2zwA7EXdtjM05P9Bl8YV_p35TQXrt4xrADEkq44Dlo,3863 +deepspeed/ops/csrc/spatial/includes/spatial_cuda_layers.h,sha256=dPh5gJbsCh2gB4_g9PSzMb1zN6LJvAGLfD7Vw_08_mc,915 +deepspeed/ops/csrc/transformer/cublas_wrappers.cu,sha256=HntTKLKThdx3Gj_jBK6gfLD1pnbVE_-wt5Yd_JTQatk,21052 +deepspeed/ops/csrc/transformer/dropout_kernels.cu,sha256=u6U7RuMioRS_aixeRL0aGvU3r7exZ-EFeE7Etgg66IM,29835 +deepspeed/ops/csrc/transformer/ds_transformer_cuda.cpp,sha256=sztj1XsEX1rbeq2lovNbYf_jZ-ExLL9qB-TLVRP_1D4,47582 +deepspeed/ops/csrc/transformer/gelu_kernels.cu,sha256=f1emIwHWorXkuIqaB9kD93L4HjzSCcQ2JL7GlE6hh6s,12191 +deepspeed/ops/csrc/transformer/general_kernels.cu,sha256=TzoOK2TqstG0MX9swk0UptK2zBgkpYwo99PeY9j0Fns,14520 +deepspeed/ops/csrc/transformer/inference/csrc/apply_rotary_pos_emb.cu,sha256=7z2HGEwCY1N-JutgeAYGsJJtXyvi36D8NdlpNaqRjcE,8594 +deepspeed/ops/csrc/transformer/inference/csrc/dequantize.cu,sha256=z7GzRO9XUJFb0DNPr7luWML9cJTYOZrR5wAQGeuStqY,4916 +deepspeed/ops/csrc/transformer/inference/csrc/gelu.cu,sha256=peyOpVxxboodFWAY0YB6vr3kakCcr8x0EJ4j7HiEY4U,28877 +deepspeed/ops/csrc/transformer/inference/csrc/layer_norm.cu,sha256=79gLSVFW19gsgFQNezyITbBImha7_oXJo5NfBloOhzU,20883 +deepspeed/ops/csrc/transformer/inference/csrc/pointwise_ops.cu,sha256=tJP7V172fEHn86xtO1H8lKzBS7CToDNofJBnOVhivEA,2476 +deepspeed/ops/csrc/transformer/inference/csrc/pt_binding.cpp,sha256=aIdgiEyTGJm_InXaO_eXQ4zH_2MMiRAx9u7vXEEnGkU,89443 +deepspeed/ops/csrc/transformer/inference/csrc/relu.cu,sha256=F_W6D8eeHtm_btn-FTyAV8FnLTDJ9H8mpkDBWpmxxDc,2318 +deepspeed/ops/csrc/transformer/inference/csrc/rms_norm.cu,sha256=0vHKVYfQQ8w-qLUD-ihfCd0jWDQ_N03QuQu0rimHrdU,10251 +deepspeed/ops/csrc/transformer/inference/csrc/softmax.cu,sha256=GY3hRUkY59BcQqe2BA0QCGw3tlN2UO4GQipRezSzJLM,27245 +deepspeed/ops/csrc/transformer/inference/csrc/transform.cu,sha256=6S1NPVL8Fw9y6wJ0rho2NVIXm64YxthNMKxSWTDsMPo,31887 +deepspeed/ops/csrc/transformer/inference/includes/inference_context.h,sha256=2-KQOUFuWdx1rOLR3rZtTmOpz9LUmY9Vs7a5qNybUzI,10531 +deepspeed/ops/csrc/transformer/inference/includes/inference_cublas_wrappers.h,sha256=AXxx5E4YDDpbKkAxozTHsNJQpIN7dSWr8CiNnpJnBLE,21657 +deepspeed/ops/csrc/transformer/inference/includes/inference_cuda_layers.h,sha256=TlXSYeyoKSWBaxuN_dDxrTMhutjQocPjKeGL_NPXIPo,9057 +deepspeed/ops/csrc/transformer/normalize_kernels.cu,sha256=u9fnXA77eqpoC1hri_w0IzC0xW7_OIG5Cs6F_UFnivs,74900 +deepspeed/ops/csrc/transformer/softmax_kernels.cu,sha256=GKmqpxpbAFlhM2p1SFMZ02phEpYMSGUFA2J9ukLovno,26758 +deepspeed/ops/csrc/transformer/transform_kernels.cu,sha256=Ob-Hae1q-KZmDxaji0Ci0nHyNuRYDvPFBF_yrnB8ZoE,22709 +deepspeed/ops/csrc/utils/py_ds_utils.cpp,sha256=jB0kJ0caZitZaw81e6LJ989yaCtCHP_tEFFXvXEfMLk,682 +deepspeed/ops/csrc/utils/tensor_cast.cpp,sha256=GZMlX8HcDDPI0ahjL4_Pf-xjurK1VlvI1dp_LUyhEXQ,797 +deepspeed/ops/csrc/utils/tensor_cast.h,sha256=OBCrxtt5tY0wmP6gd7SS8qlPVcOuYEba-PvC5MTsBSA,383 +deepspeed/ops/csrc/xpu/adagrad/cpu_adagrad.cpp,sha256=-LiTifCTt4Pr_4miMyP5-8W84FHhORWZn8mx-VZDR58,6814 +deepspeed/ops/csrc/xpu/adam/fused_adam_frontend.cpp,sha256=tu6pBcAvqcdh1ti6SB7fj8HnqFDQg3Rqbaa4xsQ2v7M,880 +deepspeed/ops/csrc/xpu/adam/multi_tensor_adam.dp.cpp,sha256=Du_BWrHEoYqoCfDb6qKxDeKXP69ZfdFEW1mt7ir5ah0,6595 +deepspeed/ops/csrc/xpu/adam/multi_tensor_apply.dp.hpp,sha256=__3PLu5FDG8vFF9fOwP45KL_yWTXYxKiNdLUYNyqnNI,9482 +deepspeed/ops/csrc/xpu/common/custom_cuda_kernel.dp.cpp,sha256=fSy_UoyFgDlpKWEDRtUqwrhaLMMUV1OG7eCVZfzcGKs,3525 +deepspeed/ops/csrc/xpu/includes/compat.h,sha256=5ivuTerIqiCaYu2HpRPzJcUtqMsAtRB7aOXel69HeZQ,336 +deepspeed/ops/csrc/xpu/includes/cpu_adagrad.h,sha256=7yZNV7T38Yvnv2JLfcdm4KdBuXyPdvT-7Plzva_5BbQ,3951 +deepspeed/ops/csrc/xpu/includes/cpu_adam.h,sha256=3a0twZXzUUnAhrEymYF8gKFoVW594B-I-Snjg6paK9Q,7956 +deepspeed/ops/csrc/xpu/includes/simd.h,sha256=tRwYTW1Xm20rDzsew6c7dfEPbCBd0tsjY26mx1VnVYk,6588 +deepspeed/ops/csrc/xpu/includes/type_shim.h,sha256=qA0DdBShoh_Wute7v52S18jvHB81xi2ADhYvn8EXRVg,7834 +deepspeed/ops/csrc/xpu/packbits/packing.cpp,sha256=oUzgPsI6tXEK56YWrSxAvf5ei289d36We-vmqTXKPuU,3292 +deepspeed/ops/deepspeed4science/__init__.py,sha256=LRmYuJYPcnqiyIuSCwsvqpD61wcScSmS7An8zq0UY4M,175 +deepspeed/ops/deepspeed4science/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/deepspeed4science/__pycache__/evoformer_attn.cpython-310.pyc,, +deepspeed/ops/deepspeed4science/evoformer_attn.py,sha256=50z8I1w4lJww3bRiBxq8PfEJxcacZw_qXk4DyTyWsSQ,4121 +deepspeed/ops/fp_quantizer/__init__.py,sha256=UxAI8a6mhBD9nD5dIfV6dAcO4qGlz8tMLBSEgVucflA,174 +deepspeed/ops/fp_quantizer/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/fp_quantizer/__pycache__/fp8_gemm.cpython-310.pyc,, +deepspeed/ops/fp_quantizer/__pycache__/fp8_gemm_triton.cpython-310.pyc,, +deepspeed/ops/fp_quantizer/__pycache__/quantize.cpython-310.pyc,, +deepspeed/ops/fp_quantizer/fp8_gemm.py,sha256=ZFcVM33qjxFg5utlPDtZ9GYMThcdFMgeqMAblhs8Fcs,939 +deepspeed/ops/fp_quantizer/fp8_gemm_triton.py,sha256=XDdslcyiSZUoOhPKKm7Hdzy60gqqkOdODqehj-iCYJY,7106 +deepspeed/ops/fp_quantizer/quantize.py,sha256=Cw19CwiaIk8kC_1dyjZSbMcYqVHEbdXNYybkVXN1xog,6481 +deepspeed/ops/gds/__init__.py,sha256=j3yUaB9pVgZKU8_d5i6-ypeWGZbpzKovc-r3ZpF5XLQ,132 +deepspeed/ops/gds/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/lamb/__init__.py,sha256=HG9WeYIi4tTtKZBuLuJNIVpLSK75djGxjXDkrgfreEk,130 +deepspeed/ops/lamb/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/lamb/__pycache__/fused_lamb.cpython-310.pyc,, +deepspeed/ops/lamb/fused_lamb.py,sha256=gZbjO38b517Ii1tEJwT0O81YwJ5bXtflM2uq57NFXtI,7815 +deepspeed/ops/lion/__init__.py,sha256=EFF2vXYA9hYPNuqBsLsd1MTFCPiOFfW1ZWBrlbxfbD8,169 +deepspeed/ops/lion/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/lion/__pycache__/cpu_lion.cpython-310.pyc,, +deepspeed/ops/lion/__pycache__/fused_lion.cpython-310.pyc,, +deepspeed/ops/lion/__pycache__/multi_tensor_apply.cpython-310.pyc,, +deepspeed/ops/lion/cpu_lion.py,sha256=mJkqnY3EG7YEDi71Fg2qD0HFiSFFFkkz84Vcx8xfG2Y,5306 +deepspeed/ops/lion/fused_lion.py,sha256=QoYzriRhYkaa_bDxIY_YGviCUe-BcNZvvNtFA4g5kLc,5548 +deepspeed/ops/lion/multi_tensor_apply.py,sha256=APt3UCnfw-nLjJOGWXC3izWRXNu-16TYhB-s213DpM0,429 +deepspeed/ops/op_builder/__init__.py,sha256=HJ00U1SrhiXDftTIq9k5hZp-CuRPZ5GWOyQerqgC21U,2005 +deepspeed/ops/op_builder/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/all_ops.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/async_io.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/builder.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/cpu_adagrad.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/cpu_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/cpu_lion.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/dc.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/evoformer_attn.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/fp_quantizer.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/fused_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/fused_lamb.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/fused_lion.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/gds.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/inference_core_ops.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/inference_cutlass_builder.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/quantizer.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/ragged_ops.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/ragged_utils.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/random_ltd.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/sparse_attn.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/spatial_inference.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/stochastic_transformer.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/transformer.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/transformer_inference.cpython-310.pyc,, +deepspeed/ops/op_builder/__pycache__/utils.cpython-310.pyc,, +deepspeed/ops/op_builder/all_ops.py,sha256=bymFEiEsBwsM_BlMOePCM-9xMGeQdk6u0ZXh2fPQo9w,1223 +deepspeed/ops/op_builder/async_io.py,sha256=EMeLJZ9GBIAjuD2AwDDsfW0GpKPCg-YrK12yqze_Y9w,4500 +deepspeed/ops/op_builder/builder.py,sha256=r3sljupH_zvXGHApweR1FuCGDEQ7hdnuePttpC6UL9E,35323 +deepspeed/ops/op_builder/cpu/__init__.py,sha256=fRbecWZV5CWo2W5bRzfweUx0C_8gjVwmj4Yb6YgIbzY,353 +deepspeed/ops/op_builder/cpu/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/op_builder/cpu/__pycache__/async_io.cpython-310.pyc,, +deepspeed/ops/op_builder/cpu/__pycache__/builder.cpython-310.pyc,, +deepspeed/ops/op_builder/cpu/__pycache__/comm.cpython-310.pyc,, +deepspeed/ops/op_builder/cpu/__pycache__/cpu_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/cpu/__pycache__/fused_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/cpu/__pycache__/no_impl.cpython-310.pyc,, +deepspeed/ops/op_builder/cpu/async_io.py,sha256=aY_ndccYdKQYQ7adRJfl3NfwY8Dj6yqn1OkBr42qn9g,3550 +deepspeed/ops/op_builder/cpu/builder.py,sha256=Bi1gqcUHoDb12ci-5Urb2QRopaQFAy5E5uNELEd4Hzg,1460 +deepspeed/ops/op_builder/cpu/comm.py,sha256=e_bVQ0AnLnZSpsuY4du3NgcpYjhpRoQOVqJu1NQmqW4,2114 +deepspeed/ops/op_builder/cpu/cpu_adam.py,sha256=rGtsjwhzzP9iMDIphGxjh5TLUBtUiEwtrnf5x5BbG08,625 +deepspeed/ops/op_builder/cpu/fused_adam.py,sha256=5LAQSEL2H4Bcc1b7LArqKCcF2Ztuq2uaABmRmI7m5Ss,546 +deepspeed/ops/op_builder/cpu/no_impl.py,sha256=OIZ3aVWXK9W8iRtgNL0pAtl033WV0hEoOuPP35UuFUc,616 +deepspeed/ops/op_builder/cpu_adagrad.py,sha256=fB81OcyxFAMVzINEi1uY1CneERg5KKS8IafOM_ICzCo,622 +deepspeed/ops/op_builder/cpu_adam.py,sha256=aBvkhLFpMxELFvFEpB57C2Fmlq_-3O3azRNEI8XglGo,635 +deepspeed/ops/op_builder/cpu_lion.py,sha256=8l8o0abx4TCn30ps_Iy40HCnymQ9ve225kp_uLRwpWk,635 +deepspeed/ops/op_builder/dc.py,sha256=VL5DLUUq2NbCrJh3Y2WrIcqZmgA1WKj9AVvGfVFDY1U,1157 +deepspeed/ops/op_builder/evoformer_attn.py,sha256=dmGaZQv-G74ckTri8yAMRwjdwomqIjjDeiu2zyCPNgU,3602 +deepspeed/ops/op_builder/fp_quantizer.py,sha256=WUFftK3MJApYc2IZDF7Cm0fBD2PmYCYprbdA23dKSIU,4130 +deepspeed/ops/op_builder/fused_adam.py,sha256=IC87eCjvRPJ2ETVdQUv0-d_yxMXhDJvbPML1AEU9cSQ,1044 +deepspeed/ops/op_builder/fused_lamb.py,sha256=Ojx9euSMUeeU7DVIFxE11w_fHERqYsZxNNtaGxnAAag,1216 +deepspeed/ops/op_builder/fused_lion.py,sha256=1ZJh92_chzH0Jupt8RO8GUOnrm6ACKLECZ5mTiV7WI0,1044 +deepspeed/ops/op_builder/gds.py,sha256=zJPklZuTb6tbi8rnswfmBAC862pwVKv-E6gcv-0QT0M,2195 +deepspeed/ops/op_builder/hpu/__init__.py,sha256=E7-aI0c0S1kZjNCKsjp7XImSfdziWHt3psvwHc82YHQ,370 +deepspeed/ops/op_builder/hpu/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/op_builder/hpu/__pycache__/builder.cpython-310.pyc,, +deepspeed/ops/op_builder/hpu/__pycache__/cpu_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/hpu/__pycache__/fp_quantizer.cpython-310.pyc,, +deepspeed/ops/op_builder/hpu/__pycache__/fused_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/hpu/__pycache__/no_impl.cpython-310.pyc,, +deepspeed/ops/op_builder/hpu/__pycache__/transformer_inference.cpython-310.pyc,, +deepspeed/ops/op_builder/hpu/builder.py,sha256=nVgIhaRZRgppzIeuYrqfvWcwpGShcK7JUDxHgosA5Cc,1391 +deepspeed/ops/op_builder/hpu/cpu_adam.py,sha256=FzHM2A3XH3dXSYy1DWA7H61swa7m4S9OLsLrbvL8Myk,681 +deepspeed/ops/op_builder/hpu/fp_quantizer.py,sha256=0278EpesoRE54UveUX1ZdlcVlqdmHkzhPk2bFg1bH-U,2947 +deepspeed/ops/op_builder/hpu/fused_adam.py,sha256=ZpP8fkXQ7KdEDtrW5XT9qQ5QlsiGQhONo5r2Mpn03SE,3498 +deepspeed/ops/op_builder/hpu/no_impl.py,sha256=cUJ87-4-I46CK3GmS-Ve52nzfMhcFayT_YZF5FWEmzY,616 +deepspeed/ops/op_builder/hpu/transformer_inference.py,sha256=dPHPLX_wdRFIbCbx4FbxeREn1AJoWOzRbO36dKmcYig,1341 +deepspeed/ops/op_builder/inference_core_ops.py,sha256=ZwfV2HtgoSUv055qWQ8xGSyjI3EUW7TiPy9Xd5DbgQ8,3835 +deepspeed/ops/op_builder/inference_cutlass_builder.py,sha256=OMvFKGBukkUTiYcNHu5CcjZ_oWNA7ifTVGlDu_wBWd0,3373 +deepspeed/ops/op_builder/mlu/__init__.py,sha256=oPAhp7_PS9EV3UCyKDFRfFifj7mJPV2YEMHlqTbVbD8,402 +deepspeed/ops/op_builder/mlu/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/op_builder/mlu/__pycache__/builder.cpython-310.pyc,, +deepspeed/ops/op_builder/mlu/__pycache__/cpu_adagrad.cpython-310.pyc,, +deepspeed/ops/op_builder/mlu/__pycache__/cpu_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/mlu/__pycache__/fused_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/mlu/__pycache__/no_impl.cpython-310.pyc,, +deepspeed/ops/op_builder/mlu/builder.py,sha256=QdxnbQZxPI98OtPEvDl186lgOat6nQIoIJsIlxYC1aw,1283 +deepspeed/ops/op_builder/mlu/cpu_adagrad.py,sha256=FCrdOCfiYHdfs1HSwhYgohE2x27jmOcFpE9TtqZRa48,565 +deepspeed/ops/op_builder/mlu/cpu_adam.py,sha256=i-Oj1EP5jeuU_7JHu7iqrs16BONW93S7RIq_ImUfyZQ,669 +deepspeed/ops/op_builder/mlu/fused_adam.py,sha256=o574SNUxqSYNXVzGAsRkXdoXpINpwQ6-rysW6SRDVMo,1137 +deepspeed/ops/op_builder/mlu/no_impl.py,sha256=j0r-HhbzJuCVzna9vLWVaDp2wE21AuHDHBN8yy7vUQU,799 +deepspeed/ops/op_builder/npu/__init__.py,sha256=utWarSvvEuL_LUk8-i4Kx2oCHJRemddoGeGfzHOCsTE,419 +deepspeed/ops/op_builder/npu/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/op_builder/npu/__pycache__/async_io.cpython-310.pyc,, +deepspeed/ops/op_builder/npu/__pycache__/builder.cpython-310.pyc,, +deepspeed/ops/op_builder/npu/__pycache__/cpu_adagrad.cpython-310.pyc,, +deepspeed/ops/op_builder/npu/__pycache__/cpu_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/npu/__pycache__/cpu_lion.cpython-310.pyc,, +deepspeed/ops/op_builder/npu/__pycache__/fused_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/npu/__pycache__/inference.cpython-310.pyc,, +deepspeed/ops/op_builder/npu/__pycache__/no_impl.cpython-310.pyc,, +deepspeed/ops/op_builder/npu/async_io.py,sha256=D6N-B3SF9bpg8y8hyHxljAdSWIcpvjhXQ3J80DJnACQ,3770 +deepspeed/ops/op_builder/npu/builder.py,sha256=k21-MqhnSCYyMpuOxbRjDWbsIcnDnA6GxWPtsmtBuC8,3200 +deepspeed/ops/op_builder/npu/cpu_adagrad.py,sha256=CM4wGBKhOHayOyzvOGX2yDIh4PgOCu6AUHXF4GKq9kE,581 +deepspeed/ops/op_builder/npu/cpu_adam.py,sha256=S5V76jfADBzNNP-CyBoTXZSWrQMLYWuos6ACygoumDE,594 +deepspeed/ops/op_builder/npu/cpu_lion.py,sha256=rDkiA8vVCMolVFWUnvN9PkrLOJlc6P_rSRNYmOUrYV8,594 +deepspeed/ops/op_builder/npu/fused_adam.py,sha256=uKjA8NKcDUqjr34ToapiXrQHD661uClj6N9equFMh_g,2142 +deepspeed/ops/op_builder/npu/inference.py,sha256=-IsUpVW106uskkegCrYzyMoed5W-RhhffawBWsXYad8,15187 +deepspeed/ops/op_builder/npu/no_impl.py,sha256=qZnbySBzMLLUrMlZV656Pdh8TGfEZKNACKdmD6uT9ho,755 +deepspeed/ops/op_builder/quantizer.py,sha256=k0kTBN9_9Ri_dyzhW_ENxYpUVhiFOeHaIdVY1W50ZPE,1037 +deepspeed/ops/op_builder/ragged_ops.py,sha256=dpipyaz2IADHNLHYC9vavFlXLEMUBJhVwWgwxagKjbI,4912 +deepspeed/ops/op_builder/ragged_utils.py,sha256=CVQGORZRAxjymG9HCuhRtnKudI2sMXd3v3MRVQhi1fQ,2727 +deepspeed/ops/op_builder/random_ltd.py,sha256=zNBXnNjckT9dZqtj4pVYHcSUu4m2govven5qh5Qp9VU,879 +deepspeed/ops/op_builder/sdaa/__init__.py,sha256=N_LPrOGu5tIJVcRRXzDm86SBtEUInPAIJLWMJWqMr1s,1811 +deepspeed/ops/op_builder/sdaa/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/op_builder/sdaa/__pycache__/builder.cpython-310.pyc,, +deepspeed/ops/op_builder/sdaa/__pycache__/cpu_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/sdaa/__pycache__/fused_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/sdaa/__pycache__/no_impl.cpython-310.pyc,, +deepspeed/ops/op_builder/sdaa/builder.py,sha256=59jhxPvqJLB-5qePUz0qXUryvSwskkFKKwaRPqRD0U4,2788 +deepspeed/ops/op_builder/sdaa/cpu_adam.py,sha256=DgbrhjECQzOkSyyfDlbrm1l1zxDKC9z3hJ0CrUVeXtQ,2175 +deepspeed/ops/op_builder/sdaa/fused_adam.py,sha256=4u_xou5pjkpTtiPMYhvmX5XuLS-RSQNwRAg9OCNvTlI,2647 +deepspeed/ops/op_builder/sdaa/no_impl.py,sha256=spQASQULrUtgzUL4Cr5TNlItdW_9TWCJwIvGIikLOUQ,2306 +deepspeed/ops/op_builder/sparse_attn.py,sha256=nled88Ns_XIbbbqEITlFF_ZgQujuuv3JKyZIEqV9GNc,3221 +deepspeed/ops/op_builder/spatial_inference.py,sha256=0c8zejdt2H05Q879i-kVCuUxnrsJo7ni0F532EGK5Gg,1599 +deepspeed/ops/op_builder/stochastic_transformer.py,sha256=nQPCaLrtuHVtJxMWIQp_8IT1iQ2XS9nQ1BJVwwAeIPI,565 +deepspeed/ops/op_builder/transformer.py,sha256=wyrj-FZTKPyXzwKOGPYDyic3w5hdtXh4dWJS_wfRdCU,1094 +deepspeed/ops/op_builder/transformer_inference.py,sha256=9R9XaLdkPxf89oZgSfZ9yTcJBpN7B4ic27vBDKTBHbY,2868 +deepspeed/ops/op_builder/utils.py,sha256=w2kP0Vg2PMV0RGbmy0erVVCYK4FN6V7A3YWhpSgJODo,455 +deepspeed/ops/op_builder/xpu/__init__.py,sha256=3WbwNA_gET5S9B3BiSglGVJB6325AKYgdsMXWYU1-7Y,421 +deepspeed/ops/op_builder/xpu/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/op_builder/xpu/__pycache__/async_io.cpython-310.pyc,, +deepspeed/ops/op_builder/xpu/__pycache__/builder.cpython-310.pyc,, +deepspeed/ops/op_builder/xpu/__pycache__/cpu_adagrad.cpython-310.pyc,, +deepspeed/ops/op_builder/xpu/__pycache__/cpu_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/xpu/__pycache__/flash_attn.cpython-310.pyc,, +deepspeed/ops/op_builder/xpu/__pycache__/fused_adam.cpython-310.pyc,, +deepspeed/ops/op_builder/xpu/__pycache__/inference.cpython-310.pyc,, +deepspeed/ops/op_builder/xpu/__pycache__/no_impl.cpython-310.pyc,, +deepspeed/ops/op_builder/xpu/__pycache__/packbits.cpython-310.pyc,, +deepspeed/ops/op_builder/xpu/async_io.py,sha256=gFV9LPwhIHNGcSk9bisKe_Z3bCXCxsPyIBFaL-sEtc0,3738 +deepspeed/ops/op_builder/xpu/builder.py,sha256=Pul5G1bF8D5fb-RpXISLJhVoZyE5vH3GNeOA8qKW_NM,5816 +deepspeed/ops/op_builder/xpu/cpu_adagrad.py,sha256=bi9U6d3UmHRYw7qkIcRNRYTuasPYoSe38ngF71ZM1RQ,576 +deepspeed/ops/op_builder/xpu/cpu_adam.py,sha256=1mUDRDd_SV_ZFxCzbt39PajtYqEjRdzHAQPzCClx7FY,627 +deepspeed/ops/op_builder/xpu/flash_attn.py,sha256=YUkFzxNQ_c8MsriNLC1pdqjk-8op60DiDIaky8u52Jk,1615 +deepspeed/ops/op_builder/xpu/fused_adam.py,sha256=yEvt078k2gqqqDjPG_D5I5Ntuz_aAqEjYgtCgjpgAjg,701 +deepspeed/ops/op_builder/xpu/inference.py,sha256=_4-VhP2iIhY9qA_tYadGlKWWtDQ95NVPooVGcyiqzIc,1134 +deepspeed/ops/op_builder/xpu/no_impl.py,sha256=ODPPWw4qSX5ITgjXYT3LHWc-JSe92eBzAIQVKpgkZRA,752 +deepspeed/ops/op_builder/xpu/packbits.py,sha256=eDSmRqT3cFETG48aeHvSHtYg5oaenw2X7WQFW4WJcM0,626 +deepspeed/ops/quantizer/__init__.py,sha256=5IdLoKmcCi6MuZNNTGADQs88dgcCUdN50WofVXDcvpI,132 +deepspeed/ops/quantizer/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/quantizer/__pycache__/quantizer.cpython-310.pyc,, +deepspeed/ops/quantizer/quantizer.py,sha256=z3dh0MRUklHnR74y662k9H31FOMU0buQOGE9uPLcUZA,1193 +deepspeed/ops/random_ltd/__init__.py,sha256=MC02456CIFtrgIYKClskYz4kDnQ9X7zRyyKNp110l58,191 +deepspeed/ops/random_ltd/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/random_ltd/__pycache__/dropping_utils.cpython-310.pyc,, +deepspeed/ops/random_ltd/dropping_utils.py,sha256=tHMRvk6WIVDoPR58HWc0OwUNzXhfn8VYd7Xw13X-r5o,4902 +deepspeed/ops/sparse_attention/__init__.py,sha256=qhwanCYxLO9eaH1cx52E4LNe_An8Nrja_8WyyoYk42I,467 +deepspeed/ops/sparse_attention/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/sparse_attention/__pycache__/bert_sparse_self_attention.cpython-310.pyc,, +deepspeed/ops/sparse_attention/__pycache__/matmul.cpython-310.pyc,, +deepspeed/ops/sparse_attention/__pycache__/softmax.cpython-310.pyc,, +deepspeed/ops/sparse_attention/__pycache__/sparse_attention_utils.cpython-310.pyc,, +deepspeed/ops/sparse_attention/__pycache__/sparse_self_attention.cpython-310.pyc,, +deepspeed/ops/sparse_attention/__pycache__/sparsity_config.cpython-310.pyc,, +deepspeed/ops/sparse_attention/bert_sparse_self_attention.py,sha256=kQM4sl9L-jNirgCWwA9U-4EWGo-AKSzQE7d2CPcCwgw,3465 +deepspeed/ops/sparse_attention/matmul.py,sha256=RYTMwGqaYRUTR7DC60Ek7PNWUc5n9sKJNgcTpzmiZfo,32948 +deepspeed/ops/sparse_attention/softmax.py,sha256=B_q4n2TP29iPfMLTnkRqgHxJ8y5cBjsTixPOO7zrrMw,11322 +deepspeed/ops/sparse_attention/sparse_attention_utils.py,sha256=XlNgnI3CdG8Kur-Ac1O5qJVQRX_lgexTzXtfarMAHb4,12300 +deepspeed/ops/sparse_attention/sparse_self_attention.py,sha256=Mgs_4NaoBoHIghomqaqnKtjbAgYs1xWlZOlZAoSaMao,6746 +deepspeed/ops/sparse_attention/sparsity_config.py,sha256=eUCYxt5QPHWXdot-Myu206H9Y5WV8HrqxvO7LPKHHKg,42463 +deepspeed/ops/sparse_attention/trsrc/__init__.py,sha256=2G0yT0H4-bH6beifS-DoGK8lQcfZ2aIqt7IFekVRxP0,1032 +deepspeed/ops/sparse_attention/trsrc/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/sparse_attention/trsrc/matmul.tr,sha256=nM6iQGzBXyKFzHqmCDjpr8xmykwEls4Te6PWVJg22Dg,6628 +deepspeed/ops/sparse_attention/trsrc/softmax_bwd.tr,sha256=C--cFHczPdwJA7wm0s66p9f_nkzDMatqu3-iBQoAlog,1923 +deepspeed/ops/sparse_attention/trsrc/softmax_fwd.tr,sha256=t4bvel9w4ilPsn24DQgzIqGNG08Pt4pfCBFua8rj9fc,4047 +deepspeed/ops/transformer/__init__.py,sha256=ADlxWPujgq5jT_XL8yuRtTg9jdQiBK-5ydzgnNwap-o,413 +deepspeed/ops/transformer/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/transformer/__pycache__/transformer.cpython-310.pyc,, +deepspeed/ops/transformer/inference/__init__.py,sha256=ENKx17yjSD2dIqD6kPq-zxY-Xae-O84-ZPGGNeFq064,315 +deepspeed/ops/transformer/inference/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/transformer/inference/__pycache__/bias_add.cpython-310.pyc,, +deepspeed/ops/transformer/inference/__pycache__/config.cpython-310.pyc,, +deepspeed/ops/transformer/inference/__pycache__/diffusers_2d_transformer.cpython-310.pyc,, +deepspeed/ops/transformer/inference/__pycache__/diffusers_attention.cpython-310.pyc,, +deepspeed/ops/transformer/inference/__pycache__/diffusers_transformer_block.cpython-310.pyc,, +deepspeed/ops/transformer/inference/__pycache__/ds_attention.cpython-310.pyc,, +deepspeed/ops/transformer/inference/__pycache__/ds_mlp.cpython-310.pyc,, +deepspeed/ops/transformer/inference/__pycache__/moe_inference.cpython-310.pyc,, +deepspeed/ops/transformer/inference/__pycache__/triton_ops.cpython-310.pyc,, +deepspeed/ops/transformer/inference/bias_add.py,sha256=x1gk_iN4pmFNsSt6jwtBXil-3no4yDhTMbi_6yrs7gY,876 +deepspeed/ops/transformer/inference/config.py,sha256=g9IEbiLEc1kZUvJ3pKvYNQEvwljcIthhupUXpSX914c,6120 +deepspeed/ops/transformer/inference/diffusers_2d_transformer.py,sha256=JVSE9B60n6XHeg37zE26W3-jzKOhxiOtJkmkCiM1nDU,236 +deepspeed/ops/transformer/inference/diffusers_attention.py,sha256=BJa7R6fYdgeqB1gBNX38hD5CqLQdcj8R0g5m4sAp8_Q,9600 +deepspeed/ops/transformer/inference/diffusers_transformer_block.py,sha256=1WHyfxnq1-JMtCZavXr0mK0lPM3vm6QNCRAqVlxwzyc,4324 +deepspeed/ops/transformer/inference/ds_attention.py,sha256=Oiy0oW5ByvWArjqzqU3qUG3eWQmhYQRf_JBwQOyJPu4,15207 +deepspeed/ops/transformer/inference/ds_mlp.py,sha256=Lr5q6pk3MrAx8D36wXhTNDlV7MsFMkFFaHb7CiKzfzo,6294 +deepspeed/ops/transformer/inference/moe_inference.py,sha256=p3Ck2gBRtnY-km8EYnFfjhuyc63rN-ZUVdBBVs97s4A,17360 +deepspeed/ops/transformer/inference/op_binding/__init__.py,sha256=JhdbkMuKVwM62BEU4aI4OscG3GGkxFKsvnttFx9vc9k,382 +deepspeed/ops/transformer/inference/op_binding/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/base.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/bias_add.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/bias_gelu.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/bias_relu.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/bias_residual.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/einsum_sec_sm_ecm.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/gated_activation.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/gelu_gemm.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/layer_norm.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/linear.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/mlp_gemm.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/moe_res_matmul.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/pad_transform.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/pre_rms_norm.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/qkv_gemm.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/residual_add.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/rms_norm.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/softmax.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/softmax_context.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/vector_add.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/vector_matmul.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/__pycache__/workspace.cpython-310.pyc,, +deepspeed/ops/transformer/inference/op_binding/base.py,sha256=YN1s4f-7BKDmvCyKkVW3DOaXz40Xxk8FrIeDU2BfmJA,536 +deepspeed/ops/transformer/inference/op_binding/bias_add.py,sha256=9zrXRrRgwfd21zT_jqYF0tJUFur3Dup3x4-WwD_4d_E,980 +deepspeed/ops/transformer/inference/op_binding/bias_gelu.py,sha256=dOuLD_SwIw0p0V45qQnosr82bCDBn1cXtMMzGUXzQ5g,1214 +deepspeed/ops/transformer/inference/op_binding/bias_relu.py,sha256=Z87V8pcsHv01FxuXk2qw-qBsEtG6CNJ0CjJ8MSBUQL0,1161 +deepspeed/ops/transformer/inference/op_binding/bias_residual.py,sha256=ZURDKvdWx8fHPr_2pe9V0d66C_8hbcPLAxgTqKeTClc,952 +deepspeed/ops/transformer/inference/op_binding/einsum_sec_sm_ecm.py,sha256=P5OTQgxyxc73DLQvyEQzzYsH7NQde25awdmSMV9HhK4,938 +deepspeed/ops/transformer/inference/op_binding/gated_activation.py,sha256=xLe2XuWm0-myxEkamkjwliNAode8-GiiCMtcpSB7iLg,1549 +deepspeed/ops/transformer/inference/op_binding/gelu_gemm.py,sha256=LZ3wUeuGrBG56n0jMFbv7-7X_TbzUnJQ6FyQpIGDHNc,2191 +deepspeed/ops/transformer/inference/op_binding/layer_norm.py,sha256=Fj2rFo3RV8Bqg1XpeX1hYVCvWiP2QReN_b9TUSypB60,2370 +deepspeed/ops/transformer/inference/op_binding/linear.py,sha256=vIYuAmj9CX0KERm9_IE7pdmf4XG4UPXkanBqqtlnhgc,2719 +deepspeed/ops/transformer/inference/op_binding/mlp_gemm.py,sha256=ZTwwz6ptQ2hRQw1Mp1_0bheTc8X7Wc8m7e5Clw7yKZ8,5745 +deepspeed/ops/transformer/inference/op_binding/moe_res_matmul.py,sha256=2w4xeLtx3u7MdD4jZQ7f1DPeWPIvbOaHVMSICVa9bDw,1002 +deepspeed/ops/transformer/inference/op_binding/pad_transform.py,sha256=dlxmmFGmY0xnUUCONADu2lH4mMnHH4KWvm1odvEnTAc,899 +deepspeed/ops/transformer/inference/op_binding/pre_rms_norm.py,sha256=L9j-2d55UW8cqUyq4g1bVxVHT7doW4ACcFdhJXjhs64,1009 +deepspeed/ops/transformer/inference/op_binding/qkv_gemm.py,sha256=vDrBmcbq1xnFcoQ7J6AGQS9X_2NHEaveC-6kPW3E1y0,4514 +deepspeed/ops/transformer/inference/op_binding/residual_add.py,sha256=1w8a4TEIRPtZKPg6z40PC-_dj0ssNDpVA-8JlfPgBFo,3084 +deepspeed/ops/transformer/inference/op_binding/rms_norm.py,sha256=S68R8QOSlT3qyzdmubRx0q6YkMC_qggDH1qPl_Hxmho,985 +deepspeed/ops/transformer/inference/op_binding/softmax.py,sha256=LE1zRTF5JvIN3YmAexhtmTiWdkk5zwdl8A8ZTkmfp1M,3105 +deepspeed/ops/transformer/inference/op_binding/softmax_context.py,sha256=A0L_4CG72sLDUK2n0dWdEVATqt5loB3UpqM66jHNVco,7527 +deepspeed/ops/transformer/inference/op_binding/vector_add.py,sha256=n_l4idWGoSjrFze9_9j_V2YnaDwnVJYzOoSCERnoVAI,882 +deepspeed/ops/transformer/inference/op_binding/vector_matmul.py,sha256=LhLI-FzBDCM9ivWOENA_NkvGAna-eqxESQBNVp7zu44,2640 +deepspeed/ops/transformer/inference/op_binding/workspace.py,sha256=-Of25xBArRF5u0zYYv2rK8SVrjWaQSDla0ztbaqv6Sw,9584 +deepspeed/ops/transformer/inference/triton/__init__.py,sha256=FVRc5cfjotwsFtcB7gkAm09lXFbNl01gaoet67x4B_k,350 +deepspeed/ops/transformer/inference/triton/__pycache__/__init__.cpython-310.pyc,, +deepspeed/ops/transformer/inference/triton/__pycache__/attention.cpython-310.pyc,, +deepspeed/ops/transformer/inference/triton/__pycache__/gelu.cpython-310.pyc,, +deepspeed/ops/transformer/inference/triton/__pycache__/layer_norm.cpython-310.pyc,, +deepspeed/ops/transformer/inference/triton/__pycache__/matmul_ext.cpython-310.pyc,, +deepspeed/ops/transformer/inference/triton/__pycache__/mlp.cpython-310.pyc,, +deepspeed/ops/transformer/inference/triton/__pycache__/ops.cpython-310.pyc,, +deepspeed/ops/transformer/inference/triton/__pycache__/residual_add.cpython-310.pyc,, +deepspeed/ops/transformer/inference/triton/__pycache__/softmax.cpython-310.pyc,, +deepspeed/ops/transformer/inference/triton/__pycache__/triton_matmul_kernel.cpython-310.pyc,, +deepspeed/ops/transformer/inference/triton/attention.py,sha256=iLSFq5MOkMXhUc9uQvpmoGo5fzYSIQPJttGNAE6BG5E,16576 +deepspeed/ops/transformer/inference/triton/gelu.py,sha256=OAViD-qU6B50KhjQgg-Y3leuEKP6gW2OkX7MX_VGDko,1152 +deepspeed/ops/transformer/inference/triton/layer_norm.py,sha256=q5Xt2ov4z5DonYRBnWevaUYTFwJU5qIrJc6RVDtLaPw,7512 +deepspeed/ops/transformer/inference/triton/matmul_ext.py,sha256=6020-OGlpSZv8ytcKDnctmwggKCQ9XdLcOjjgkK3R-0,16028 +deepspeed/ops/transformer/inference/triton/mlp.py,sha256=c4yRQv3GPzEKcaxMFwkms4q2eF47ewemkYcnuhu0iec,4225 +deepspeed/ops/transformer/inference/triton/ops.py,sha256=5YLnOEbqn-GydhgR26Qw2z5zZRNDDxQQ_O3oNQyng5w,3804 +deepspeed/ops/transformer/inference/triton/residual_add.py,sha256=PYcAlDC8yz00LzJ8eXsJKoMmxjD_nMM0X5PbK8w_J70,3118 +deepspeed/ops/transformer/inference/triton/softmax.py,sha256=2q_qNmffFM4l2UgsAFKrJmAVrID95KK1rcg5dEzBmco,3208 +deepspeed/ops/transformer/inference/triton/triton_matmul_kernel.py,sha256=8pa5U6ZJphYxJ3oX2cPdxdYilmuKf8w_lvt1A8-pjbg,12570 +deepspeed/ops/transformer/inference/triton_ops.py,sha256=lR82VswoMZ9b2Axbck_eF5GCOczUmvKTDAT1W8bVH1w,5487 +deepspeed/ops/transformer/transformer.py,sha256=1UMs9JNjlK3mD5PJi3iER9QARUO2_a5h0sZAY0Rmo24,20600 +deepspeed/pipe/__init__.py,sha256=ddCnO6IbTpGzlAIgfEbGg6RPCz70HNN-MwTjCpTsqLI,164 +deepspeed/pipe/__pycache__/__init__.cpython-310.pyc,, +deepspeed/profiling/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/profiling/__pycache__/__init__.cpython-310.pyc,, +deepspeed/profiling/__pycache__/config.cpython-310.pyc,, +deepspeed/profiling/__pycache__/constants.cpython-310.pyc,, +deepspeed/profiling/config.py,sha256=OzJuTtxVvm62T1iA5eXocvP8xQQXHNtzcCdqISLTTKM,1959 +deepspeed/profiling/constants.py,sha256=QaZFzozkotv71aArhiwtQJ13DNDGDU1xC0Jm0KrVH08,1243 +deepspeed/profiling/flops_profiler/__init__.py,sha256=NCZv_Ktz4sFmmfpBwGKVVbbKrI36VQHwPapvLlWUUxE,120 +deepspeed/profiling/flops_profiler/__pycache__/__init__.cpython-310.pyc,, +deepspeed/profiling/flops_profiler/__pycache__/profiler.cpython-310.pyc,, +deepspeed/profiling/flops_profiler/profiler.py,sha256=NtUZADA8U0-9PioVuNKZnOIFPnQ_-dSlxWhdguJwaXw,51818 +deepspeed/runtime/__init__.py,sha256=4I9UpQ5vMRU5SYSF_dW9FJDEnBq4m_0SuwtVQ92lGaA,95 +deepspeed/runtime/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/__pycache__/base_optimizer.cpython-310.pyc,, +deepspeed/runtime/__pycache__/bf16_optimizer.cpython-310.pyc,, +deepspeed/runtime/__pycache__/compiler.cpython-310.pyc,, +deepspeed/runtime/__pycache__/config.cpython-310.pyc,, +deepspeed/runtime/__pycache__/config_utils.cpython-310.pyc,, +deepspeed/runtime/__pycache__/constants.cpython-310.pyc,, +deepspeed/runtime/__pycache__/dataloader.cpython-310.pyc,, +deepspeed/runtime/__pycache__/eigenvalue.cpython-310.pyc,, +deepspeed/runtime/__pycache__/engine.cpython-310.pyc,, +deepspeed/runtime/__pycache__/hybrid_engine.cpython-310.pyc,, +deepspeed/runtime/__pycache__/lr_schedules.cpython-310.pyc,, +deepspeed/runtime/__pycache__/precision_config.cpython-310.pyc,, +deepspeed/runtime/__pycache__/progressive_layer_drop.cpython-310.pyc,, +deepspeed/runtime/__pycache__/quantize.cpython-310.pyc,, +deepspeed/runtime/__pycache__/sparse_tensor.cpython-310.pyc,, +deepspeed/runtime/__pycache__/state_dict_factory.cpython-310.pyc,, +deepspeed/runtime/__pycache__/torch_autocast.cpython-310.pyc,, +deepspeed/runtime/__pycache__/utils.cpython-310.pyc,, +deepspeed/runtime/__pycache__/weight_quantizer.cpython-310.pyc,, +deepspeed/runtime/activation_checkpointing/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/runtime/activation_checkpointing/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/activation_checkpointing/__pycache__/checkpointing.cpython-310.pyc,, +deepspeed/runtime/activation_checkpointing/__pycache__/config.cpython-310.pyc,, +deepspeed/runtime/activation_checkpointing/checkpointing.py,sha256=AfqMBlOkP0DYiNJyqej_9zbzQdaP3aomJ0scwxn6CZs,44313 +deepspeed/runtime/activation_checkpointing/config.py,sha256=YQkhHYT6Qr2GJl_KusLU0pUJ-rbLLKKj0Zo7LkeZkAg,3988 +deepspeed/runtime/base_optimizer.py,sha256=qqzmVfz_Xdah_QogN3Nz6hiT8NUJacyGjX1uh7TTzQ0,3475 +deepspeed/runtime/bf16_optimizer.py,sha256=hvKo9HanSTB9kClFJEiBQDBNFVa5mS89pl0z2M_ZkxE,25572 +deepspeed/runtime/checkpoint_engine/__init__.py,sha256=LaqB9nJrtf9qqyhJ0Eyi_mDvYXOkGVK8-XIuxWWBVuw,420 +deepspeed/runtime/checkpoint_engine/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/checkpoint_engine/__pycache__/checkpoint_engine.cpython-310.pyc,, +deepspeed/runtime/checkpoint_engine/__pycache__/decoupled_checkpoint_engine.cpython-310.pyc,, +deepspeed/runtime/checkpoint_engine/__pycache__/fast_checkpoint_engine.cpython-310.pyc,, +deepspeed/runtime/checkpoint_engine/__pycache__/nebula_checkpoint_engine.cpython-310.pyc,, +deepspeed/runtime/checkpoint_engine/__pycache__/torch_checkpoint_engine.cpython-310.pyc,, +deepspeed/runtime/checkpoint_engine/__pycache__/utils.cpython-310.pyc,, +deepspeed/runtime/checkpoint_engine/checkpoint_engine.py,sha256=82YbbHCTWv898UU075AemsYniypgmHLGUVLq6C-HwPU,1257 +deepspeed/runtime/checkpoint_engine/decoupled_checkpoint_engine.py,sha256=1dnj6UgtfkaOqEq3NpwwwOE_3pnv9mmRuFSKTSoC1NY,5808 +deepspeed/runtime/checkpoint_engine/fast_checkpoint_engine.py,sha256=Cr-9Px8TGIqVtrErYArLFZ84ueqKIGIXDh4KGO4njTg,1761 +deepspeed/runtime/checkpoint_engine/nebula_checkpoint_engine.py,sha256=5FDE7UUwM_xbB6StNjtynSyEfLpljyx0FwmWi4u9sME,5126 +deepspeed/runtime/checkpoint_engine/torch_checkpoint_engine.py,sha256=UDWWw1hnn9oYWS0lRa_JP9Mp80Yic0iJ2g6Y7t5SZtg,1764 +deepspeed/runtime/checkpoint_engine/utils.py,sha256=ABeLEQGo9amYK0h_jkTytjENf2ybNkyBKphmyVdQyEI,1820 +deepspeed/runtime/comm/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/runtime/comm/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/comm/__pycache__/coalesced_collectives.cpython-310.pyc,, +deepspeed/runtime/comm/__pycache__/compressed.cpython-310.pyc,, +deepspeed/runtime/comm/__pycache__/hccl.cpython-310.pyc,, +deepspeed/runtime/comm/__pycache__/mpi.cpython-310.pyc,, +deepspeed/runtime/comm/__pycache__/nccl.cpython-310.pyc,, +deepspeed/runtime/comm/coalesced_collectives.py,sha256=2tdtNA68uONSuqZull-4T_p2eao2wChCLsRUIxM0BR8,11545 +deepspeed/runtime/comm/compressed.py,sha256=yflJJlYN2Wb47h6PUOnUJfmRIS8TJdjMi3LwVAbhvGA,5661 +deepspeed/runtime/comm/hccl.py,sha256=O1XqCH-DdBbJMlTxRkR2l5OUpMQYWXLhkgyZlPwbdCk,5176 +deepspeed/runtime/comm/mpi.py,sha256=m3-XVeEL6F5ln5krGaExKcBj1Qlxg7y_XwvvVo8APcU,10076 +deepspeed/runtime/comm/nccl.py,sha256=kAvrk95McM3accR11f2Grg1spe1kwP8UmC000zemsJw,7584 +deepspeed/runtime/compiler.py,sha256=WKhCrWTU1HVO3hPIYjJy92GHtuRGJDdIET5bYQFR780,1725 +deepspeed/runtime/compression/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/runtime/compression/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/compression/__pycache__/cupy.cpython-310.pyc,, +deepspeed/runtime/compression/cupy.py,sha256=gCQ7N_EE7TRgZmRaGRHuhGAXrtEkvfbAnJTBMGO-pXI,691 +deepspeed/runtime/config.py,sha256=tuJDGKzsXBASgi1tFUImqeGxsZ2ip3nDie_rYgLB-Zo,40536 +deepspeed/runtime/config_utils.py,sha256=Mjlms4PInPnKZct1xW-ceuulZCvhsspm1SwnuqTs_SE,8460 +deepspeed/runtime/constants.py,sha256=eizwrjKZpfT0qFgAcYBZNF4MGPexhMYgM20o7QZgkFc,15571 +deepspeed/runtime/data_pipeline/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/runtime/data_pipeline/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/__pycache__/config.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/__pycache__/constants.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/__pycache__/curriculum_scheduler.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/config.py,sha256=9GGQNeHTR-bog6aBpcxCm65jc5QO2Y3KWNMZoPfK4gs,7721 +deepspeed/runtime/data_pipeline/constants.py,sha256=x4rApIpDlX1zPViP01wMiKqGNPSiqzhzNZ4VlQ7zFp4,5702 +deepspeed/runtime/data_pipeline/curriculum_scheduler.py,sha256=g6Heo7I5L8mpyKJXSH8t1SwO2Lxk9jMfOIMbYDksV00,10025 +deepspeed/runtime/data_pipeline/data_routing/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/runtime/data_pipeline/data_routing/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/data_routing/__pycache__/basic_layer.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/data_routing/__pycache__/helper.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/data_routing/__pycache__/scheduler.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/data_routing/__pycache__/utils.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/data_routing/basic_layer.py,sha256=QvJuYJRbFTZSkKovN5Z7PVQP5o_fNC-2QgYqrWGy8ns,5638 +deepspeed/runtime/data_pipeline/data_routing/helper.py,sha256=mFWBiepSdAAejDPSaIV6sXeKpRlfXoWFGF0CUmiDACY,1282 +deepspeed/runtime/data_pipeline/data_routing/scheduler.py,sha256=8ddlbZJ3RO0btPKmurh3288Lk1CIoMxRWHcrxlVMZkY,4638 +deepspeed/runtime/data_pipeline/data_routing/utils.py,sha256=ZGrHeImPXbVhLwOMQGWowXEK5YpIGXfC8i3RHVF4NAI,955 +deepspeed/runtime/data_pipeline/data_sampling/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/runtime/data_pipeline/data_sampling/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/data_sampling/__pycache__/data_analyzer.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/data_sampling/__pycache__/data_sampler.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/data_sampling/__pycache__/indexed_dataset.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/data_sampling/__pycache__/utils.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/data_sampling/__pycache__/variable_batch_size_and_lr.cpython-310.pyc,, +deepspeed/runtime/data_pipeline/data_sampling/data_analyzer.py,sha256=eN9-LeQE66qratDmZ4vyZL1jPs2Fc-M4UM0HXG4Yg-Q,48993 +deepspeed/runtime/data_pipeline/data_sampling/data_sampler.py,sha256=PJNw94Zc0L31DRk4DH9vkXXGvXPMqtisnbF79gFY7UY,19894 +deepspeed/runtime/data_pipeline/data_sampling/indexed_dataset.py,sha256=QQdA40qmTehZqhBDX2oGeJI8m1r8c6Jc5Hx-ygPa-j8,21449 +deepspeed/runtime/data_pipeline/data_sampling/utils.py,sha256=5QtKBV_54fKL9I7ydqF40uBaLkUpcX5LFeYVz2KY_r4,1690 +deepspeed/runtime/data_pipeline/data_sampling/variable_batch_size_and_lr.py,sha256=o-fRgO_JiJ32nQh0rrh-Zz_uGfqsfqHzPsHTkl_KpLE,24311 +deepspeed/runtime/dataloader.py,sha256=rNiiaHzYOU3hp8OAiC2g9tFWPwACJTMhJ_EVrxvP5OU,6977 +deepspeed/runtime/domino/__init__.py,sha256=4I9UpQ5vMRU5SYSF_dW9FJDEnBq4m_0SuwtVQ92lGaA,95 +deepspeed/runtime/domino/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/domino/__pycache__/async_linear.cpython-310.pyc,, +deepspeed/runtime/domino/__pycache__/transformer.cpython-310.pyc,, +deepspeed/runtime/domino/async_linear.py,sha256=4kDO5n0bLu71BuksABfK4bbi5Opp95JfeYJrOqRJuWQ,4576 +deepspeed/runtime/domino/transformer.py,sha256=xU4nTT-xy52S33rM2tuWlAjG6GxRpNEPQ2lf4MVT0Lg,25278 +deepspeed/runtime/eigenvalue.py,sha256=5SEkMHwnGPqRFyU7ymuoO6cOkRcDDcJnjsnp2wXa2JM,5872 +deepspeed/runtime/engine.py,sha256=tu30PwPILQwNeaqWJWzS83WtaH9SS7rG3jG1WfkKIg4,193515 +deepspeed/runtime/fp16/__init__.py,sha256=Dzme9x1YQb-Ru4gzsCNsv8r19zkGyqwPwPwPgZzief4,140 +deepspeed/runtime/fp16/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/fp16/__pycache__/fused_optimizer.cpython-310.pyc,, +deepspeed/runtime/fp16/__pycache__/loss_scaler.cpython-310.pyc,, +deepspeed/runtime/fp16/__pycache__/unfused_optimizer.cpython-310.pyc,, +deepspeed/runtime/fp16/fused_optimizer.py,sha256=w1ke8oaojnHEz-w8kVr_dy_3d72x1lnFt_z4UY18he4,22322 +deepspeed/runtime/fp16/loss_scaler.py,sha256=7E3yzMJEFzbKcIR2aeUx9-9GrB6_2sGv5wOX8NZ6ezY,11652 +deepspeed/runtime/fp16/onebit/__init__.py,sha256=8hnDOpk71O_EekeWgWIJ4CnbtYt67dre4flu5hoES-4,186 +deepspeed/runtime/fp16/onebit/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/fp16/onebit/__pycache__/adam.cpython-310.pyc,, +deepspeed/runtime/fp16/onebit/__pycache__/lamb.cpython-310.pyc,, +deepspeed/runtime/fp16/onebit/__pycache__/zoadam.cpython-310.pyc,, +deepspeed/runtime/fp16/onebit/adam.py,sha256=jRCUSTzBhvo1xPD5XvRxmHni8R2JXMBGL1_P7udBmpc,15694 +deepspeed/runtime/fp16/onebit/lamb.py,sha256=jMZBvuwG8GxoF4tD22ECq9jZyl8_XBcAKnDLyS0GDHA,23556 +deepspeed/runtime/fp16/onebit/zoadam.py,sha256=z3Pc2u9LyJXm5hCfQjAzj_wp72RJP8In6LYv5KyrChA,19554 +deepspeed/runtime/fp16/unfused_optimizer.py,sha256=qB2H34wcs1q05TqhyrA5wPYmC-RJduqjxswGA1rs3Ng,18029 +deepspeed/runtime/hybrid_engine.py,sha256=rfP2MpmnB-DqCiSRjwKyklcl78RtGNwFOQa-PiutlWU,20965 +deepspeed/runtime/lr_schedules.py,sha256=eDap8FWaxsp6M-VSxi07NWLKWCwEwyqbOLIQb6frQxU,39074 +deepspeed/runtime/model_checkpointing/__init__.py,sha256=9EfmW2osMT_aBTR0xOE5NadoNHh7mETjTJxf5lnz-4k,173 +deepspeed/runtime/model_checkpointing/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/model_checkpointing/__pycache__/config.cpython-310.pyc,, +deepspeed/runtime/model_checkpointing/__pycache__/constants.cpython-310.pyc,, +deepspeed/runtime/model_checkpointing/__pycache__/data_parallel_writer_factory.cpython-310.pyc,, +deepspeed/runtime/model_checkpointing/__pycache__/utils.cpython-310.pyc,, +deepspeed/runtime/model_checkpointing/__pycache__/writer_factory.cpython-310.pyc,, +deepspeed/runtime/model_checkpointing/config.py,sha256=jFcd7SQFwXj7JqWgpJHXinDN5j9X2-73U4aWd4BBFaU,2992 +deepspeed/runtime/model_checkpointing/constants.py,sha256=JJ2jAMNPTnWuFwr6JJelgayVBryCcIQhpXlxXBga0zg,2258 +deepspeed/runtime/model_checkpointing/data_parallel_writer_factory.py,sha256=hsW-v_h4VfZM8gbn57EBK-Ov4noINYpnKAku8GD9x6Y,10732 +deepspeed/runtime/model_checkpointing/utils.py,sha256=C78G1D098fb3ffbstnGkhcO8CEw2pUbGq9NLtxMSsl8,3275 +deepspeed/runtime/model_checkpointing/writer_factory.py,sha256=SaFlZX0cRvXrIVKTg_qSmOOd7NK3mYvpQx4NYdpPQz4,4930 +deepspeed/runtime/pipe/__init__.py,sha256=4Xc534VEOCSKYVc-ImMFLx5eTmPB_URVgWqweehFsVI,195 +deepspeed/runtime/pipe/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/pipe/__pycache__/engine.cpython-310.pyc,, +deepspeed/runtime/pipe/__pycache__/module.cpython-310.pyc,, +deepspeed/runtime/pipe/__pycache__/p2p.cpython-310.pyc,, +deepspeed/runtime/pipe/__pycache__/schedule.cpython-310.pyc,, +deepspeed/runtime/pipe/__pycache__/topology.cpython-310.pyc,, +deepspeed/runtime/pipe/engine.py,sha256=Rg89Sfun9GELR8Ob4OrClFyEmi8fMn6cOrOn6-WQ8eQ,63690 +deepspeed/runtime/pipe/module.py,sha256=FfLY5D8DZVGy0kEtcTs7Pqmc4U3V7Lq6TdPwZyRMYy0,30551 +deepspeed/runtime/pipe/p2p.py,sha256=t4K_Q2W2YI4K5bPOF1iqC5z7Ytyj4K1vHKA3gcSioBU,5344 +deepspeed/runtime/pipe/schedule.py,sha256=ZDzAeTPZTaYt77Wi2KSkycfuwneBGFsa7V6afQAzHo4,15546 +deepspeed/runtime/pipe/topology.py,sha256=Y_bQMiTNioGRb4twzWfsxlSkZc0iepbTbs9yvx7H-Cs,17698 +deepspeed/runtime/precision_config.py,sha256=HGZxwS8lygvliSHrtYLBl3B2eJmW37j7crRYzRt4UMQ,3612 +deepspeed/runtime/progressive_layer_drop.py,sha256=5zb3-BrMbRxYZL5lk1FtvaOknMi31xT1refDkgSMQbQ,1353 +deepspeed/runtime/quantize.py,sha256=OmimrTJV_RoVG8TLVdjcQR91kSatQCNdFkPf3U39fsk,7699 +deepspeed/runtime/sequence_parallel/__init__.py,sha256=2BSwDtJJ1v4uCmvYlXq0W17dbHPGC4v0-Xv0IfmtQb0,99 +deepspeed/runtime/sequence_parallel/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/sequence_parallel/__pycache__/parallel_state_sp.cpython-310.pyc,, +deepspeed/runtime/sequence_parallel/__pycache__/ulysses_sp.cpython-310.pyc,, +deepspeed/runtime/sequence_parallel/parallel_state_sp.py,sha256=SfNF80rWVfeByWao4GcT3wCBOibzTNqZlFi4GO8l2Fs,3677 +deepspeed/runtime/sequence_parallel/ulysses_sp.py,sha256=kSWH4eW6Hr-Aj-Z518SxCKXsXC98g1vgmTQnUYeNIVY,64116 +deepspeed/runtime/sparse_tensor.py,sha256=QBoplFkxSYLqE9Jfj8mN2qNLMMpVpEgpP3nuIsobpRg,2466 +deepspeed/runtime/state_dict_factory.py,sha256=Zp2mJbIySbYgp2cl5pFvLkpROOOqX9eqGnG1b3d0kD0,18177 +deepspeed/runtime/swap_tensor/__init__.py,sha256=OiMgE9mkwbOsv3gC91ENJRAHc7zlVfMXXhn4pxN7zcc,134 +deepspeed/runtime/swap_tensor/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/swap_tensor/__pycache__/aio_config.cpython-310.pyc,, +deepspeed/runtime/swap_tensor/__pycache__/async_swapper.cpython-310.pyc,, +deepspeed/runtime/swap_tensor/__pycache__/constants.cpython-310.pyc,, +deepspeed/runtime/swap_tensor/__pycache__/optimizer_utils.cpython-310.pyc,, +deepspeed/runtime/swap_tensor/__pycache__/partitioned_optimizer_swapper.cpython-310.pyc,, +deepspeed/runtime/swap_tensor/__pycache__/partitioned_param_swapper.cpython-310.pyc,, +deepspeed/runtime/swap_tensor/__pycache__/pipelined_optimizer_swapper.cpython-310.pyc,, +deepspeed/runtime/swap_tensor/__pycache__/utils.cpython-310.pyc,, +deepspeed/runtime/swap_tensor/aio_config.py,sha256=CaII3llLuEecOBceu7MuaqUtyFyzxARYE3Fspyj7DQ0,1643 +deepspeed/runtime/swap_tensor/async_swapper.py,sha256=RlwlqBbkHa3nviVT4nrYzZ4s05nPch_dNMHTZcFOdxw,6349 +deepspeed/runtime/swap_tensor/constants.py,sha256=lmGtuinGdUVZ8lEgvKKj0dVLUuiLRQTfPaEhxxWPg7E,700 +deepspeed/runtime/swap_tensor/optimizer_utils.py,sha256=MTQIkQ9zT0C6gpyS8zeB9et40z1BXFnh1e1z5FJBaZA,21403 +deepspeed/runtime/swap_tensor/partitioned_optimizer_swapper.py,sha256=0-t1zOLHC_PMm5BnllC0BwjToFpYvBYYHby_ALmIOYw,10629 +deepspeed/runtime/swap_tensor/partitioned_param_swapper.py,sha256=U3Oopxi88nOvXx7wkDAt8eGnk1fuvIoV5ClhDce6QHc,18791 +deepspeed/runtime/swap_tensor/pipelined_optimizer_swapper.py,sha256=AiFBvtcln-TrI3pSnaQzLTvaVTNBfiObA9_muL35R_M,11346 +deepspeed/runtime/swap_tensor/utils.py,sha256=Yrgbja_CefG1juXQVS-_KfnUu4vmP5Cw-r0p_fJ8DgU,7820 +deepspeed/runtime/tensor_parallel/__init__.py,sha256=BwFdU2Aa2rmV01DRps6I9hRTGsOiBzsTGH7oaEoeUDE,198 +deepspeed/runtime/tensor_parallel/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/tensor_parallel/__pycache__/config.cpython-310.pyc,, +deepspeed/runtime/tensor_parallel/__pycache__/tp_manager.cpython-310.pyc,, +deepspeed/runtime/tensor_parallel/config.py,sha256=rcNLwzsdTG1wInLFF1JI916VjJFYxhuoFuZZNWiZfmE,2860 +deepspeed/runtime/tensor_parallel/tp_manager.py,sha256=nPEscixH1Of-jaTsyrXzGo3qSHK0gaWDpfnVbZzbDhg,2539 +deepspeed/runtime/torch_autocast.py,sha256=ERSLVErVR1athv4vKhGPebAT8oBWzS1oQexgfCPt7wA,3638 +deepspeed/runtime/utils.py,sha256=4WyJnDEc7k87RxQco63P1PIxikNABzB62aYN7E0v34A,46087 +deepspeed/runtime/weight_quantizer.py,sha256=-aS6hTZnMcc3ckoYaCI6MiHybwy2NZo5AqEZ8rT7G_o,7027 +deepspeed/runtime/zero/__init__.py,sha256=3hNEJ-32gAoqG24c00D2lrC4WY0qFQPr0AbNSqZIazU,611 +deepspeed/runtime/zero/__pycache__/__init__.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/config.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/contiguous_memory_allocator.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/linear.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/mics.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/mics_utils.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/offload_config.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/offload_states.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/parameter_offload.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/partition_parameters.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/partitioned_param_coordinator.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/partitioned_param_profiler.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/stage3.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/stage_1_and_2.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/test.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/tiling.cpython-310.pyc,, +deepspeed/runtime/zero/__pycache__/utils.cpython-310.pyc,, +deepspeed/runtime/zero/config.py,sha256=tc0rFE0llwvM70WIyiLTc4iVWH8WZZhE0PftZp98AF8,14345 +deepspeed/runtime/zero/contiguous_memory_allocator.py,sha256=r1J-KRJqO_Aa4Rqlsb1cgtHPa4mpnxm_zrd19FzZVQU,10923 +deepspeed/runtime/zero/linear.py,sha256=yCqBF0da6ECXHyf0GLT9Wsso0HanizkWc-Rx32WWDgo,7994 +deepspeed/runtime/zero/mics.py,sha256=mKc4itpTLX3oQqciDiHHHmgF7cd_IRjOjgMv_5chk7E,22339 +deepspeed/runtime/zero/mics_utils.py,sha256=GWmBVESUd699i4YjCKScOwAwlwwFGs0si1dmPAeMz60,7500 +deepspeed/runtime/zero/offload_config.py,sha256=NN8h1kMNtH61GrSbnLTED2JnjzwGYL8v3IozOrSDIPI,3359 +deepspeed/runtime/zero/offload_states.py,sha256=WThDC2EhyaOArrehBNr-_VUNPc4hY5ZeTIXKsGh9Qhg,2956 +deepspeed/runtime/zero/parameter_offload.py,sha256=ObXU8MT1znWw8gg5vlIREvTzXqiGPIta4QBzVdAwFc8,26426 +deepspeed/runtime/zero/partition_parameters.py,sha256=UAyLRWwGjmz0QB0MyabXqAh6TUaO3BPGyyv0j9Mj_w8,110409 +deepspeed/runtime/zero/partitioned_param_coordinator.py,sha256=SRtBMg4UDdCvWTdlOTWsu3BgkQtTR1EE6DJTgWNmg8w,29511 +deepspeed/runtime/zero/partitioned_param_profiler.py,sha256=xTXgTnu1ew215TluuO2HvP78hZNbqPEhzJBieeOChf8,1801 +deepspeed/runtime/zero/stage3.py,sha256=T-fj76aHYDM9HOidMFOO6KgBxVIlq0pLQlVGn1NL10Y,155232 +deepspeed/runtime/zero/stage_1_and_2.py,sha256=A1t4UUHgHKBQ3KyTybM1LpvZ2qVdhtscfGWf9Sb73hE,126808 +deepspeed/runtime/zero/test.py,sha256=j7zu8Fnxm5_Wmp-aaUiQicPD1EUL2JGpODeSARFSZVs,2727 +deepspeed/runtime/zero/tiling.py,sha256=Uz6IgvJPAQ4GICXj4E509p_BIDvnczWVtBT7EjZU9rg,11727 +deepspeed/runtime/zero/utils.py,sha256=mpmn-5rjhKzd9BsXWoAf6cB90Ed4PnpynoCRgE0tBAs,5868 +deepspeed/sequence/__init__.py,sha256=4I9UpQ5vMRU5SYSF_dW9FJDEnBq4m_0SuwtVQ92lGaA,95 +deepspeed/sequence/__pycache__/__init__.cpython-310.pyc,, +deepspeed/sequence/__pycache__/cross_entropy.cpython-310.pyc,, +deepspeed/sequence/__pycache__/fpdt_layer.cpython-310.pyc,, +deepspeed/sequence/__pycache__/layer.cpython-310.pyc,, +deepspeed/sequence/cross_entropy.py,sha256=-YW4WYSHzZf_42OVu4WJuAp53GQAEnjDfIvGugAKA5k,2140 +deepspeed/sequence/fpdt_layer.py,sha256=MqPtxPXHxa5Gp6RFBIrQujHyJ_Qu8cJWH0g4gOEIGU0,56246 +deepspeed/sequence/layer.py,sha256=o-qB87cWrjJjf6152Xh0JWiCOzO-Kg6HCrAtEytHeW4,20516 +deepspeed/utils/__init__.py,sha256=n_hu2LhHCYt0EIvkNxEVJrSqyL_AhGWqQAAN6TkYPdg,1407 +deepspeed/utils/__pycache__/__init__.cpython-310.pyc,, +deepspeed/utils/__pycache__/bwc.cpython-310.pyc,, +deepspeed/utils/__pycache__/comms_logging.cpython-310.pyc,, +deepspeed/utils/__pycache__/config.cpython-310.pyc,, +deepspeed/utils/__pycache__/debug.cpython-310.pyc,, +deepspeed/utils/__pycache__/exceptions.cpython-310.pyc,, +deepspeed/utils/__pycache__/groups.cpython-310.pyc,, +deepspeed/utils/__pycache__/init_on_device.cpython-310.pyc,, +deepspeed/utils/__pycache__/logging.cpython-310.pyc,, +deepspeed/utils/__pycache__/mixed_precision_linkage.cpython-310.pyc,, +deepspeed/utils/__pycache__/numa.cpython-310.pyc,, +deepspeed/utils/__pycache__/nvtx.cpython-310.pyc,, +deepspeed/utils/__pycache__/tensor_fragment.cpython-310.pyc,, +deepspeed/utils/__pycache__/timer.cpython-310.pyc,, +deepspeed/utils/__pycache__/torch.cpython-310.pyc,, +deepspeed/utils/__pycache__/types.cpython-310.pyc,, +deepspeed/utils/__pycache__/z3_leaf_module.cpython-310.pyc,, +deepspeed/utils/__pycache__/zero_to_fp32.cpython-310.pyc,, +deepspeed/utils/bwc.py,sha256=EbdKjvtp_6XwUc3XhCyPxYfIqe9v5I23ynXGG39aAcY,3970 +deepspeed/utils/comms_logging.py,sha256=9eaWVJ5m8j59TzuA3n45-IVZFzGMOwVKQm48sU3uCH4,16080 +deepspeed/utils/config.py,sha256=O4l-6Da_8RWBeyJ4tZpKqJPCLNtYK7eMVaqsVCbQkRU,1344 +deepspeed/utils/debug.py,sha256=ilbE4Z7cz3Ww0DR7TzP6vPQbJjyZFaq-ayO6FagSTAo,4936 +deepspeed/utils/exceptions.py,sha256=h4J_9uk3HmKG8LdiSO8DKYzOmIYrD1MGwtZD1nA-Q3g,144 +deepspeed/utils/groups.py,sha256=CNBrWWq3J9m04nnpZjaViZYNnZEtEp89LViE1LVnDog,30800 +deepspeed/utils/init_on_device.py,sha256=Q4RFeRMi7PYGBHfTKh3E63X8IZquheDWYKbT56aOuuQ,3004 +deepspeed/utils/logging.py,sha256=Khqk1omHmqqcYJgI88YW-JxY0zFnNYZ7QAfLAsixX40,5016 +deepspeed/utils/mixed_precision_linkage.py,sha256=OAzA20w_15nLUurjltJRTkGC__-aNx68KWrf5cbYfRg,2484 +deepspeed/utils/numa.py,sha256=ggGkhc8gXvuzkSqpl7JzmQ4K3H9Y44pnaq9XUAgBq-Q,7174 +deepspeed/utils/nvtx.py,sha256=XeqDmA2fdAUTH1FtFQwEr0K5Y7-gTWe9ZqLSp5qttec,746 +deepspeed/utils/tensor_fragment.py,sha256=6uEsYS1-eq8iz9iViCx8LzA231jIpSPIjSFC5QUdQ_o,17039 +deepspeed/utils/timer.py,sha256=V1iXDzsYWmuE0l1hoMKditAJMr1y8sShO5MZOfyF9r0,10742 +deepspeed/utils/torch.py,sha256=xxVsKHjO8HqeDJeffSed1TiP36IAlMn8lsakLuj-MGo,888 +deepspeed/utils/types.py,sha256=IcEwFod7RqLTRCGf7lr2sbKhFRf1an1KvtQ98nswwNw,434 +deepspeed/utils/z3_leaf_module.py,sha256=9eVQubxo8VxE-hBkrI3qxrddUKFoOBTUL4SCKCQiBe8,4758 +deepspeed/utils/zero_to_fp32.py,sha256=Rkl1ZczytKix9vGMg0EELzdJYFqUM1yB9p3xvSaK9k8,33272 diff --git a/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/REQUESTED b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/WHEEL b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..e7fa31b6f3f78deb1022c1f7927f07d4d16da822 --- /dev/null +++ b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.9.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/entry_points.txt b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..cfa0d2ef62101b531baf7ac75575e64b2c0cb6a2 --- /dev/null +++ b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[pytest_randomly.random_seeder] +deepspeed = deepspeed.runtime.utils:set_random_seed diff --git a/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/licenses/LICENSE b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/licenses/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/top_level.txt b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..0f2c819def28c8c4109156014fde317020d4ddb9 --- /dev/null +++ b/lib/python3.1/site-packages/deepspeed-0.17.4.dist-info/top_level.txt @@ -0,0 +1 @@ +deepspeed diff --git a/lib/python3.1/site-packages/diskcache/__init__.py b/lib/python3.1/site-packages/diskcache/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7757d6601e278ee8879debe76b971cb3d9f1187a --- /dev/null +++ b/lib/python3.1/site-packages/diskcache/__init__.py @@ -0,0 +1,68 @@ +""" +DiskCache API Reference +======================= + +The :doc:`tutorial` provides a helpful walkthrough of most methods. +""" + +from .core import ( + DEFAULT_SETTINGS, + ENOVAL, + EVICTION_POLICY, + UNKNOWN, + Cache, + Disk, + EmptyDirWarning, + JSONDisk, + Timeout, + UnknownFileWarning, +) +from .fanout import FanoutCache +from .persistent import Deque, Index +from .recipes import ( + Averager, + BoundedSemaphore, + Lock, + RLock, + barrier, + memoize_stampede, + throttle, +) + +__all__ = [ + 'Averager', + 'BoundedSemaphore', + 'Cache', + 'DEFAULT_SETTINGS', + 'Deque', + 'Disk', + 'ENOVAL', + 'EVICTION_POLICY', + 'EmptyDirWarning', + 'FanoutCache', + 'Index', + 'JSONDisk', + 'Lock', + 'RLock', + 'Timeout', + 'UNKNOWN', + 'UnknownFileWarning', + 'barrier', + 'memoize_stampede', + 'throttle', +] + +try: + from .djangocache import DjangoCache # noqa + + __all__.append('DjangoCache') +except Exception: # pylint: disable=broad-except # pragma: no cover + # Django not installed or not setup so ignore. + pass + +__title__ = 'diskcache' +__version__ = '5.6.3' +__build__ = 0x050603 +__author__ = 'Grant Jenks' +__license__ = 'Apache 2.0' +__copyright__ = 'Copyright 2016-2023 Grant Jenks' diff --git a/lib/python3.1/site-packages/diskcache/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/site-packages/diskcache/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f54173e82d1dbf798639bfd70640f937122a94ea Binary files /dev/null and b/lib/python3.1/site-packages/diskcache/__pycache__/__init__.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/diskcache/__pycache__/cli.cpython-310.pyc b/lib/python3.1/site-packages/diskcache/__pycache__/cli.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..32805584da18010349676e34b917e993ba7f6b9f Binary files /dev/null and b/lib/python3.1/site-packages/diskcache/__pycache__/cli.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/diskcache/__pycache__/core.cpython-310.pyc b/lib/python3.1/site-packages/diskcache/__pycache__/core.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c7e11495e104842bfed285e0db6c7b30fdd957bd Binary files /dev/null and b/lib/python3.1/site-packages/diskcache/__pycache__/core.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/diskcache/__pycache__/djangocache.cpython-310.pyc b/lib/python3.1/site-packages/diskcache/__pycache__/djangocache.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..257b96beec23ef95c575b14cddf2447abec659cf Binary files /dev/null and b/lib/python3.1/site-packages/diskcache/__pycache__/djangocache.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/diskcache/__pycache__/fanout.cpython-310.pyc b/lib/python3.1/site-packages/diskcache/__pycache__/fanout.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a23c6a02c4234381a4df4458e0c0ff804e117bbc Binary files /dev/null and b/lib/python3.1/site-packages/diskcache/__pycache__/fanout.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/diskcache/__pycache__/persistent.cpython-310.pyc b/lib/python3.1/site-packages/diskcache/__pycache__/persistent.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b21e623fe5a3f26a19035fba1cb75ffa7abbd6cf Binary files /dev/null and b/lib/python3.1/site-packages/diskcache/__pycache__/persistent.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/diskcache/__pycache__/recipes.cpython-310.pyc b/lib/python3.1/site-packages/diskcache/__pycache__/recipes.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..619e49da77cc24cf93a1b286bc1a4345e2a4168b Binary files /dev/null and b/lib/python3.1/site-packages/diskcache/__pycache__/recipes.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/diskcache/cli.py b/lib/python3.1/site-packages/diskcache/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..6a39f601fe116f0a1f4e7a1866df2cb18a0ebdb4 --- /dev/null +++ b/lib/python3.1/site-packages/diskcache/cli.py @@ -0,0 +1 @@ +"""Command line interface to disk cache.""" diff --git a/lib/python3.1/site-packages/diskcache/core.py b/lib/python3.1/site-packages/diskcache/core.py new file mode 100644 index 0000000000000000000000000000000000000000..c7c84860ea853fe9232e8ee52eaaa053b00a3a87 --- /dev/null +++ b/lib/python3.1/site-packages/diskcache/core.py @@ -0,0 +1,2452 @@ +"""Core disk and file backed cache API. +""" + +import codecs +import contextlib as cl +import errno +import functools as ft +import io +import json +import os +import os.path as op +import pickle +import pickletools +import sqlite3 +import struct +import tempfile +import threading +import time +import warnings +import zlib + + +def full_name(func): + """Return full name of `func` by adding the module and function name.""" + return func.__module__ + '.' + func.__qualname__ + + +class Constant(tuple): + """Pretty display of immutable constant.""" + + def __new__(cls, name): + return tuple.__new__(cls, (name,)) + + def __repr__(self): + return '%s' % self[0] + + +DBNAME = 'cache.db' +ENOVAL = Constant('ENOVAL') +UNKNOWN = Constant('UNKNOWN') + +MODE_NONE = 0 +MODE_RAW = 1 +MODE_BINARY = 2 +MODE_TEXT = 3 +MODE_PICKLE = 4 + +DEFAULT_SETTINGS = { + 'statistics': 0, # False + 'tag_index': 0, # False + 'eviction_policy': 'least-recently-stored', + 'size_limit': 2**30, # 1gb + 'cull_limit': 10, + 'sqlite_auto_vacuum': 1, # FULL + 'sqlite_cache_size': 2**13, # 8,192 pages + 'sqlite_journal_mode': 'wal', + 'sqlite_mmap_size': 2**26, # 64mb + 'sqlite_synchronous': 1, # NORMAL + 'disk_min_file_size': 2**15, # 32kb + 'disk_pickle_protocol': pickle.HIGHEST_PROTOCOL, +} + +METADATA = { + 'count': 0, + 'size': 0, + 'hits': 0, + 'misses': 0, +} + +EVICTION_POLICY = { + 'none': { + 'init': None, + 'get': None, + 'cull': None, + }, + 'least-recently-stored': { + 'init': ( + 'CREATE INDEX IF NOT EXISTS Cache_store_time ON' + ' Cache (store_time)' + ), + 'get': None, + 'cull': 'SELECT {fields} FROM Cache ORDER BY store_time LIMIT ?', + }, + 'least-recently-used': { + 'init': ( + 'CREATE INDEX IF NOT EXISTS Cache_access_time ON' + ' Cache (access_time)' + ), + 'get': 'access_time = {now}', + 'cull': 'SELECT {fields} FROM Cache ORDER BY access_time LIMIT ?', + }, + 'least-frequently-used': { + 'init': ( + 'CREATE INDEX IF NOT EXISTS Cache_access_count ON' + ' Cache (access_count)' + ), + 'get': 'access_count = access_count + 1', + 'cull': 'SELECT {fields} FROM Cache ORDER BY access_count LIMIT ?', + }, +} + + +class Disk: + """Cache key and value serialization for SQLite database and files.""" + + def __init__(self, directory, min_file_size=0, pickle_protocol=0): + """Initialize disk instance. + + :param str directory: directory path + :param int min_file_size: minimum size for file use + :param int pickle_protocol: pickle protocol for serialization + + """ + self._directory = directory + self.min_file_size = min_file_size + self.pickle_protocol = pickle_protocol + + def hash(self, key): + """Compute portable hash for `key`. + + :param key: key to hash + :return: hash value + + """ + mask = 0xFFFFFFFF + disk_key, _ = self.put(key) + type_disk_key = type(disk_key) + + if type_disk_key is sqlite3.Binary: + return zlib.adler32(disk_key) & mask + elif type_disk_key is str: + return zlib.adler32(disk_key.encode('utf-8')) & mask # noqa + elif type_disk_key is int: + return disk_key % mask + else: + assert type_disk_key is float + return zlib.adler32(struct.pack('!d', disk_key)) & mask + + def put(self, key): + """Convert `key` to fields key and raw for Cache table. + + :param key: key to convert + :return: (database key, raw boolean) pair + + """ + # pylint: disable=unidiomatic-typecheck + type_key = type(key) + + if type_key is bytes: + return sqlite3.Binary(key), True + elif ( + (type_key is str) + or ( + type_key is int + and -9223372036854775808 <= key <= 9223372036854775807 + ) + or (type_key is float) + ): + return key, True + else: + data = pickle.dumps(key, protocol=self.pickle_protocol) + result = pickletools.optimize(data) + return sqlite3.Binary(result), False + + def get(self, key, raw): + """Convert fields `key` and `raw` from Cache table to key. + + :param key: database key to convert + :param bool raw: flag indicating raw database storage + :return: corresponding Python key + + """ + # pylint: disable=unidiomatic-typecheck + if raw: + return bytes(key) if type(key) is sqlite3.Binary else key + else: + return pickle.load(io.BytesIO(key)) + + def store(self, value, read, key=UNKNOWN): + """Convert `value` to fields size, mode, filename, and value for Cache + table. + + :param value: value to convert + :param bool read: True when value is file-like object + :param key: key for item (default UNKNOWN) + :return: (size, mode, filename, value) tuple for Cache table + + """ + # pylint: disable=unidiomatic-typecheck + type_value = type(value) + min_file_size = self.min_file_size + + if ( + (type_value is str and len(value) < min_file_size) + or ( + type_value is int + and -9223372036854775808 <= value <= 9223372036854775807 + ) + or (type_value is float) + ): + return 0, MODE_RAW, None, value + elif type_value is bytes: + if len(value) < min_file_size: + return 0, MODE_RAW, None, sqlite3.Binary(value) + else: + filename, full_path = self.filename(key, value) + self._write(full_path, io.BytesIO(value), 'xb') + return len(value), MODE_BINARY, filename, None + elif type_value is str: + filename, full_path = self.filename(key, value) + self._write(full_path, io.StringIO(value), 'x', 'UTF-8') + size = op.getsize(full_path) + return size, MODE_TEXT, filename, None + elif read: + reader = ft.partial(value.read, 2**22) + filename, full_path = self.filename(key, value) + iterator = iter(reader, b'') + size = self._write(full_path, iterator, 'xb') + return size, MODE_BINARY, filename, None + else: + result = pickle.dumps(value, protocol=self.pickle_protocol) + + if len(result) < min_file_size: + return 0, MODE_PICKLE, None, sqlite3.Binary(result) + else: + filename, full_path = self.filename(key, value) + self._write(full_path, io.BytesIO(result), 'xb') + return len(result), MODE_PICKLE, filename, None + + def _write(self, full_path, iterator, mode, encoding=None): + full_dir, _ = op.split(full_path) + + for count in range(1, 11): + with cl.suppress(OSError): + os.makedirs(full_dir) + + try: + # Another cache may have deleted the directory before + # the file could be opened. + writer = open(full_path, mode, encoding=encoding) + except OSError: + if count == 10: + # Give up after 10 tries to open the file. + raise + continue + + with writer: + size = 0 + for chunk in iterator: + size += len(chunk) + writer.write(chunk) + return size + + def fetch(self, mode, filename, value, read): + """Convert fields `mode`, `filename`, and `value` from Cache table to + value. + + :param int mode: value mode raw, binary, text, or pickle + :param str filename: filename of corresponding value + :param value: database value + :param bool read: when True, return an open file handle + :return: corresponding Python value + :raises: IOError if the value cannot be read + + """ + # pylint: disable=unidiomatic-typecheck,consider-using-with + if mode == MODE_RAW: + return bytes(value) if type(value) is sqlite3.Binary else value + elif mode == MODE_BINARY: + if read: + return open(op.join(self._directory, filename), 'rb') + else: + with open(op.join(self._directory, filename), 'rb') as reader: + return reader.read() + elif mode == MODE_TEXT: + full_path = op.join(self._directory, filename) + with open(full_path, 'r', encoding='UTF-8') as reader: + return reader.read() + elif mode == MODE_PICKLE: + if value is None: + with open(op.join(self._directory, filename), 'rb') as reader: + return pickle.load(reader) + else: + return pickle.load(io.BytesIO(value)) + + def filename(self, key=UNKNOWN, value=UNKNOWN): + """Return filename and full-path tuple for file storage. + + Filename will be a randomly generated 28 character hexadecimal string + with ".val" suffixed. Two levels of sub-directories will be used to + reduce the size of directories. On older filesystems, lookups in + directories with many files may be slow. + + The default implementation ignores the `key` and `value` parameters. + + In some scenarios, for example :meth:`Cache.push + `, the `key` or `value` may not be known when the + item is stored in the cache. + + :param key: key for item (default UNKNOWN) + :param value: value for item (default UNKNOWN) + + """ + # pylint: disable=unused-argument + hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8') + sub_dir = op.join(hex_name[:2], hex_name[2:4]) + name = hex_name[4:] + '.val' + filename = op.join(sub_dir, name) + full_path = op.join(self._directory, filename) + return filename, full_path + + def remove(self, file_path): + """Remove a file given by `file_path`. + + This method is cross-thread and cross-process safe. If an OSError + occurs, it is suppressed. + + :param str file_path: relative path to file + + """ + full_path = op.join(self._directory, file_path) + full_dir, _ = op.split(full_path) + + # Suppress OSError that may occur if two caches attempt to delete the + # same file or directory at the same time. + + with cl.suppress(OSError): + os.remove(full_path) + + with cl.suppress(OSError): + os.removedirs(full_dir) + + +class JSONDisk(Disk): + """Cache key and value using JSON serialization with zlib compression.""" + + def __init__(self, directory, compress_level=1, **kwargs): + """Initialize JSON disk instance. + + Keys and values are compressed using the zlib library. The + `compress_level` is an integer from 0 to 9 controlling the level of + compression; 1 is fastest and produces the least compression, 9 is + slowest and produces the most compression, and 0 is no compression. + + :param str directory: directory path + :param int compress_level: zlib compression level (default 1) + :param kwargs: super class arguments + + """ + self.compress_level = compress_level + super().__init__(directory, **kwargs) + + def put(self, key): + json_bytes = json.dumps(key).encode('utf-8') + data = zlib.compress(json_bytes, self.compress_level) + return super().put(data) + + def get(self, key, raw): + data = super().get(key, raw) + return json.loads(zlib.decompress(data).decode('utf-8')) + + def store(self, value, read, key=UNKNOWN): + if not read: + json_bytes = json.dumps(value).encode('utf-8') + value = zlib.compress(json_bytes, self.compress_level) + return super().store(value, read, key=key) + + def fetch(self, mode, filename, value, read): + data = super().fetch(mode, filename, value, read) + if not read: + data = json.loads(zlib.decompress(data).decode('utf-8')) + return data + + +class Timeout(Exception): + """Database timeout expired.""" + + +class UnknownFileWarning(UserWarning): + """Warning used by Cache.check for unknown files.""" + + +class EmptyDirWarning(UserWarning): + """Warning used by Cache.check for empty directories.""" + + +def args_to_key(base, args, kwargs, typed, ignore): + """Create cache key out of function arguments. + + :param tuple base: base of key + :param tuple args: function arguments + :param dict kwargs: function keyword arguments + :param bool typed: include types in cache key + :param set ignore: positional or keyword args to ignore + :return: cache key tuple + + """ + args = tuple(arg for index, arg in enumerate(args) if index not in ignore) + key = base + args + (None,) + + if kwargs: + kwargs = {key: val for key, val in kwargs.items() if key not in ignore} + sorted_items = sorted(kwargs.items()) + + for item in sorted_items: + key += item + + if typed: + key += tuple(type(arg) for arg in args) + + if kwargs: + key += tuple(type(value) for _, value in sorted_items) + + return key + + +class Cache: + """Disk and file backed cache.""" + + def __init__(self, directory=None, timeout=60, disk=Disk, **settings): + """Initialize cache instance. + + :param str directory: cache directory + :param float timeout: SQLite connection timeout + :param disk: Disk type or subclass for serialization + :param settings: any of DEFAULT_SETTINGS + + """ + try: + assert issubclass(disk, Disk) + except (TypeError, AssertionError): + raise ValueError('disk must subclass diskcache.Disk') from None + + if directory is None: + directory = tempfile.mkdtemp(prefix='diskcache-') + directory = str(directory) + directory = op.expanduser(directory) + directory = op.expandvars(directory) + + self._directory = directory + self._timeout = 0 # Manually handle retries during initialization. + self._local = threading.local() + self._txn_id = None + + if not op.isdir(directory): + try: + os.makedirs(directory, 0o755) + except OSError as error: + if error.errno != errno.EEXIST: + raise EnvironmentError( + error.errno, + 'Cache directory "%s" does not exist' + ' and could not be created' % self._directory, + ) from None + + sql = self._sql_retry + + # Setup Settings table. + + try: + current_settings = dict( + sql('SELECT key, value FROM Settings').fetchall() + ) + except sqlite3.OperationalError: + current_settings = {} + + sets = DEFAULT_SETTINGS.copy() + sets.update(current_settings) + sets.update(settings) + + for key in METADATA: + sets.pop(key, None) + + # Chance to set pragmas before any tables are created. + + for key, value in sorted(sets.items()): + if key.startswith('sqlite_'): + self.reset(key, value, update=False) + + sql( + 'CREATE TABLE IF NOT EXISTS Settings (' + ' key TEXT NOT NULL UNIQUE,' + ' value)' + ) + + # Setup Disk object (must happen after settings initialized). + + kwargs = { + key[5:]: value + for key, value in sets.items() + if key.startswith('disk_') + } + self._disk = disk(directory, **kwargs) + + # Set cached attributes: updates settings and sets pragmas. + + for key, value in sets.items(): + query = 'INSERT OR REPLACE INTO Settings VALUES (?, ?)' + sql(query, (key, value)) + self.reset(key, value) + + for key, value in METADATA.items(): + query = 'INSERT OR IGNORE INTO Settings VALUES (?, ?)' + sql(query, (key, value)) + self.reset(key) + + ((self._page_size,),) = sql('PRAGMA page_size').fetchall() + + # Setup Cache table. + + sql( + 'CREATE TABLE IF NOT EXISTS Cache (' + ' rowid INTEGER PRIMARY KEY,' + ' key BLOB,' + ' raw INTEGER,' + ' store_time REAL,' + ' expire_time REAL,' + ' access_time REAL,' + ' access_count INTEGER DEFAULT 0,' + ' tag BLOB,' + ' size INTEGER DEFAULT 0,' + ' mode INTEGER DEFAULT 0,' + ' filename TEXT,' + ' value BLOB)' + ) + + sql( + 'CREATE UNIQUE INDEX IF NOT EXISTS Cache_key_raw ON' + ' Cache(key, raw)' + ) + + sql( + 'CREATE INDEX IF NOT EXISTS Cache_expire_time ON' + ' Cache (expire_time)' + ) + + query = EVICTION_POLICY[self.eviction_policy]['init'] + + if query is not None: + sql(query) + + # Use triggers to keep Metadata updated. + + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_count_insert' + ' AFTER INSERT ON Cache FOR EACH ROW BEGIN' + ' UPDATE Settings SET value = value + 1' + ' WHERE key = "count"; END' + ) + + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_count_delete' + ' AFTER DELETE ON Cache FOR EACH ROW BEGIN' + ' UPDATE Settings SET value = value - 1' + ' WHERE key = "count"; END' + ) + + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_size_insert' + ' AFTER INSERT ON Cache FOR EACH ROW BEGIN' + ' UPDATE Settings SET value = value + NEW.size' + ' WHERE key = "size"; END' + ) + + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_size_update' + ' AFTER UPDATE ON Cache FOR EACH ROW BEGIN' + ' UPDATE Settings' + ' SET value = value + NEW.size - OLD.size' + ' WHERE key = "size"; END' + ) + + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_size_delete' + ' AFTER DELETE ON Cache FOR EACH ROW BEGIN' + ' UPDATE Settings SET value = value - OLD.size' + ' WHERE key = "size"; END' + ) + + # Create tag index if requested. + + if self.tag_index: # pylint: disable=no-member + self.create_tag_index() + else: + self.drop_tag_index() + + # Close and re-open database connection with given timeout. + + self.close() + self._timeout = timeout + self._sql # pylint: disable=pointless-statement + + @property + def directory(self): + """Cache directory.""" + return self._directory + + @property + def timeout(self): + """SQLite connection timeout value in seconds.""" + return self._timeout + + @property + def disk(self): + """Disk used for serialization.""" + return self._disk + + @property + def _con(self): + # Check process ID to support process forking. If the process + # ID changes, close the connection and update the process ID. + + local_pid = getattr(self._local, 'pid', None) + pid = os.getpid() + + if local_pid != pid: + self.close() + self._local.pid = pid + + con = getattr(self._local, 'con', None) + + if con is None: + con = self._local.con = sqlite3.connect( + op.join(self._directory, DBNAME), + timeout=self._timeout, + isolation_level=None, + ) + + # Some SQLite pragmas work on a per-connection basis so + # query the Settings table and reset the pragmas. The + # Settings table may not exist so catch and ignore the + # OperationalError that may occur. + + try: + select = 'SELECT key, value FROM Settings' + settings = con.execute(select).fetchall() + except sqlite3.OperationalError: + pass + else: + for key, value in settings: + if key.startswith('sqlite_'): + self.reset(key, value, update=False) + + return con + + @property + def _sql(self): + return self._con.execute + + @property + def _sql_retry(self): + sql = self._sql + + # 2018-11-01 GrantJ - Some SQLite builds/versions handle + # the SQLITE_BUSY return value and connection parameter + # "timeout" differently. For a more reliable duration, + # manually retry the statement for 60 seconds. Only used + # by statements which modify the database and do not use + # a transaction (like those in ``__init__`` or ``reset``). + # See Issue #85 for and tests/issue_85.py for more details. + + def _execute_with_retry(statement, *args, **kwargs): + start = time.time() + while True: + try: + return sql(statement, *args, **kwargs) + except sqlite3.OperationalError as exc: + if str(exc) != 'database is locked': + raise + diff = time.time() - start + if diff > 60: + raise + time.sleep(0.001) + + return _execute_with_retry + + @cl.contextmanager + def transact(self, retry=False): + """Context manager to perform a transaction by locking the cache. + + While the cache is locked, no other write operation is permitted. + Transactions should therefore be as short as possible. Read and write + operations performed in a transaction are atomic. Read operations may + occur concurrent to a transaction. + + Transactions may be nested and may not be shared between threads. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + >>> cache = Cache() + >>> with cache.transact(): # Atomically increment two keys. + ... _ = cache.incr('total', 123.4) + ... _ = cache.incr('count', 1) + >>> with cache.transact(): # Atomically calculate average. + ... average = cache['total'] / cache['count'] + >>> average + 123.4 + + :param bool retry: retry if database timeout occurs (default False) + :return: context manager for use in `with` statement + :raises Timeout: if database timeout occurs + + """ + with self._transact(retry=retry): + yield + + @cl.contextmanager + def _transact(self, retry=False, filename=None): + sql = self._sql + filenames = [] + _disk_remove = self._disk.remove + tid = threading.get_ident() + txn_id = self._txn_id + + if tid == txn_id: + begin = False + else: + while True: + try: + sql('BEGIN IMMEDIATE') + begin = True + self._txn_id = tid + break + except sqlite3.OperationalError: + if retry: + continue + if filename is not None: + _disk_remove(filename) + raise Timeout from None + + try: + yield sql, filenames.append + except BaseException: + if begin: + assert self._txn_id == tid + self._txn_id = None + sql('ROLLBACK') + raise + else: + if begin: + assert self._txn_id == tid + self._txn_id = None + sql('COMMIT') + for name in filenames: + if name is not None: + _disk_remove(name) + + def set(self, key, value, expire=None, read=False, tag=None, retry=False): + """Set `key` and `value` item in cache. + + When `read` is `True`, `value` should be a file-like object opened + for reading in binary mode. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param key: key for item + :param value: value for item + :param float expire: seconds until item expires + (default None, no expiry) + :param bool read: read value as bytes from file (default False) + :param str tag: text to associate with key (default None) + :param bool retry: retry if database timeout occurs (default False) + :return: True if item was set + :raises Timeout: if database timeout occurs + + """ + now = time.time() + db_key, raw = self._disk.put(key) + expire_time = None if expire is None else now + expire + size, mode, filename, db_value = self._disk.store(value, read, key=key) + columns = (expire_time, tag, size, mode, filename, db_value) + + # The order of SELECT, UPDATE, and INSERT is important below. + # + # Typical cache usage pattern is: + # + # value = cache.get(key) + # if value is None: + # value = expensive_calculation() + # cache.set(key, value) + # + # Cache.get does not evict expired keys to avoid writes during lookups. + # Commonly used/expired keys will therefore remain in the cache making + # an UPDATE the preferred path. + # + # The alternative is to assume the key is not present by first trying + # to INSERT and then handling the IntegrityError that occurs from + # violating the UNIQUE constraint. This optimistic approach was + # rejected based on the common cache usage pattern. + # + # INSERT OR REPLACE aka UPSERT is not used because the old filename may + # need cleanup. + + with self._transact(retry, filename) as (sql, cleanup): + rows = sql( + 'SELECT rowid, filename FROM Cache' + ' WHERE key = ? AND raw = ?', + (db_key, raw), + ).fetchall() + + if rows: + ((rowid, old_filename),) = rows + cleanup(old_filename) + self._row_update(rowid, now, columns) + else: + self._row_insert(db_key, raw, now, columns) + + self._cull(now, sql, cleanup) + + return True + + def __setitem__(self, key, value): + """Set corresponding `value` for `key` in cache. + + :param key: key for item + :param value: value for item + :return: corresponding value + :raises KeyError: if key is not found + + """ + self.set(key, value, retry=True) + + def _row_update(self, rowid, now, columns): + sql = self._sql + expire_time, tag, size, mode, filename, value = columns + sql( + 'UPDATE Cache SET' + ' store_time = ?,' + ' expire_time = ?,' + ' access_time = ?,' + ' access_count = ?,' + ' tag = ?,' + ' size = ?,' + ' mode = ?,' + ' filename = ?,' + ' value = ?' + ' WHERE rowid = ?', + ( + now, # store_time + expire_time, + now, # access_time + 0, # access_count + tag, + size, + mode, + filename, + value, + rowid, + ), + ) + + def _row_insert(self, key, raw, now, columns): + sql = self._sql + expire_time, tag, size, mode, filename, value = columns + sql( + 'INSERT INTO Cache(' + ' key, raw, store_time, expire_time, access_time,' + ' access_count, tag, size, mode, filename, value' + ') VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + ( + key, + raw, + now, # store_time + expire_time, + now, # access_time + 0, # access_count + tag, + size, + mode, + filename, + value, + ), + ) + + def _cull(self, now, sql, cleanup, limit=None): + cull_limit = self.cull_limit if limit is None else limit + + if cull_limit == 0: + return + + # Evict expired keys. + + select_expired_template = ( + 'SELECT %s FROM Cache' + ' WHERE expire_time IS NOT NULL AND expire_time < ?' + ' ORDER BY expire_time LIMIT ?' + ) + + select_expired = select_expired_template % 'filename' + rows = sql(select_expired, (now, cull_limit)).fetchall() + + if rows: + delete_expired = 'DELETE FROM Cache WHERE rowid IN (%s)' % ( + select_expired_template % 'rowid' + ) + sql(delete_expired, (now, cull_limit)) + + for (filename,) in rows: + cleanup(filename) + + cull_limit -= len(rows) + + if cull_limit == 0: + return + + # Evict keys by policy. + + select_policy = EVICTION_POLICY[self.eviction_policy]['cull'] + + if select_policy is None or self.volume() < self.size_limit: + return + + select_filename = select_policy.format(fields='filename', now=now) + rows = sql(select_filename, (cull_limit,)).fetchall() + + if rows: + delete = 'DELETE FROM Cache WHERE rowid IN (%s)' % ( + select_policy.format(fields='rowid', now=now) + ) + sql(delete, (cull_limit,)) + + for (filename,) in rows: + cleanup(filename) + + def touch(self, key, expire=None, retry=False): + """Touch `key` in cache and update `expire` time. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param key: key for item + :param float expire: seconds until item expires + (default None, no expiry) + :param bool retry: retry if database timeout occurs (default False) + :return: True if key was touched + :raises Timeout: if database timeout occurs + + """ + now = time.time() + db_key, raw = self._disk.put(key) + expire_time = None if expire is None else now + expire + + with self._transact(retry) as (sql, _): + rows = sql( + 'SELECT rowid, expire_time FROM Cache' + ' WHERE key = ? AND raw = ?', + (db_key, raw), + ).fetchall() + + if rows: + ((rowid, old_expire_time),) = rows + + if old_expire_time is None or old_expire_time > now: + sql( + 'UPDATE Cache SET expire_time = ? WHERE rowid = ?', + (expire_time, rowid), + ) + return True + + return False + + def add(self, key, value, expire=None, read=False, tag=None, retry=False): + """Add `key` and `value` item to cache. + + Similar to `set`, but only add to cache if key not present. + + Operation is atomic. Only one concurrent add operation for a given key + will succeed. + + When `read` is `True`, `value` should be a file-like object opened + for reading in binary mode. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param key: key for item + :param value: value for item + :param float expire: seconds until the key expires + (default None, no expiry) + :param bool read: read value as bytes from file (default False) + :param str tag: text to associate with key (default None) + :param bool retry: retry if database timeout occurs (default False) + :return: True if item was added + :raises Timeout: if database timeout occurs + + """ + now = time.time() + db_key, raw = self._disk.put(key) + expire_time = None if expire is None else now + expire + size, mode, filename, db_value = self._disk.store(value, read, key=key) + columns = (expire_time, tag, size, mode, filename, db_value) + + with self._transact(retry, filename) as (sql, cleanup): + rows = sql( + 'SELECT rowid, filename, expire_time FROM Cache' + ' WHERE key = ? AND raw = ?', + (db_key, raw), + ).fetchall() + + if rows: + ((rowid, old_filename, old_expire_time),) = rows + + if old_expire_time is None or old_expire_time > now: + cleanup(filename) + return False + + cleanup(old_filename) + self._row_update(rowid, now, columns) + else: + self._row_insert(db_key, raw, now, columns) + + self._cull(now, sql, cleanup) + + return True + + def incr(self, key, delta=1, default=0, retry=False): + """Increment value by delta for item with key. + + If key is missing and default is None then raise KeyError. Else if key + is missing and default is not None then use default for value. + + Operation is atomic. All concurrent increment operations will be + counted individually. + + Assumes value may be stored in a SQLite column. Most builds that target + machines with 64-bit pointer widths will support 64-bit signed + integers. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param key: key for item + :param int delta: amount to increment (default 1) + :param int default: value if key is missing (default 0) + :param bool retry: retry if database timeout occurs (default False) + :return: new value for item + :raises KeyError: if key is not found and default is None + :raises Timeout: if database timeout occurs + + """ + now = time.time() + db_key, raw = self._disk.put(key) + select = ( + 'SELECT rowid, expire_time, filename, value FROM Cache' + ' WHERE key = ? AND raw = ?' + ) + + with self._transact(retry) as (sql, cleanup): + rows = sql(select, (db_key, raw)).fetchall() + + if not rows: + if default is None: + raise KeyError(key) + + value = default + delta + columns = (None, None) + self._disk.store( + value, False, key=key + ) + self._row_insert(db_key, raw, now, columns) + self._cull(now, sql, cleanup) + return value + + ((rowid, expire_time, filename, value),) = rows + + if expire_time is not None and expire_time < now: + if default is None: + raise KeyError(key) + + value = default + delta + columns = (None, None) + self._disk.store( + value, False, key=key + ) + self._row_update(rowid, now, columns) + self._cull(now, sql, cleanup) + cleanup(filename) + return value + + value += delta + + columns = 'store_time = ?, value = ?' + update_column = EVICTION_POLICY[self.eviction_policy]['get'] + + if update_column is not None: + columns += ', ' + update_column.format(now=now) + + update = 'UPDATE Cache SET %s WHERE rowid = ?' % columns + sql(update, (now, value, rowid)) + + return value + + def decr(self, key, delta=1, default=0, retry=False): + """Decrement value by delta for item with key. + + If key is missing and default is None then raise KeyError. Else if key + is missing and default is not None then use default for value. + + Operation is atomic. All concurrent decrement operations will be + counted individually. + + Unlike Memcached, negative values are supported. Value may be + decremented below zero. + + Assumes value may be stored in a SQLite column. Most builds that target + machines with 64-bit pointer widths will support 64-bit signed + integers. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param key: key for item + :param int delta: amount to decrement (default 1) + :param int default: value if key is missing (default 0) + :param bool retry: retry if database timeout occurs (default False) + :return: new value for item + :raises KeyError: if key is not found and default is None + :raises Timeout: if database timeout occurs + + """ + return self.incr(key, -delta, default, retry) + + def get( + self, + key, + default=None, + read=False, + expire_time=False, + tag=False, + retry=False, + ): + """Retrieve value from cache. If `key` is missing, return `default`. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param key: key for item + :param default: value to return if key is missing (default None) + :param bool read: if True, return file handle to value + (default False) + :param bool expire_time: if True, return expire_time in tuple + (default False) + :param bool tag: if True, return tag in tuple (default False) + :param bool retry: retry if database timeout occurs (default False) + :return: value for item or default if key not found + :raises Timeout: if database timeout occurs + + """ + db_key, raw = self._disk.put(key) + update_column = EVICTION_POLICY[self.eviction_policy]['get'] + select = ( + 'SELECT rowid, expire_time, tag, mode, filename, value' + ' FROM Cache WHERE key = ? AND raw = ?' + ' AND (expire_time IS NULL OR expire_time > ?)' + ) + + if expire_time and tag: + default = (default, None, None) + elif expire_time or tag: + default = (default, None) + + if not self.statistics and update_column is None: + # Fast path, no transaction necessary. + + rows = self._sql(select, (db_key, raw, time.time())).fetchall() + + if not rows: + return default + + ((rowid, db_expire_time, db_tag, mode, filename, db_value),) = rows + + try: + value = self._disk.fetch(mode, filename, db_value, read) + except IOError: + # Key was deleted before we could retrieve result. + return default + + else: # Slow path, transaction required. + cache_hit = ( + 'UPDATE Settings SET value = value + 1 WHERE key = "hits"' + ) + cache_miss = ( + 'UPDATE Settings SET value = value + 1 WHERE key = "misses"' + ) + + with self._transact(retry) as (sql, _): + rows = sql(select, (db_key, raw, time.time())).fetchall() + + if not rows: + if self.statistics: + sql(cache_miss) + return default + + ( + (rowid, db_expire_time, db_tag, mode, filename, db_value), + ) = rows # noqa: E127 + + try: + value = self._disk.fetch(mode, filename, db_value, read) + except IOError: + # Key was deleted before we could retrieve result. + if self.statistics: + sql(cache_miss) + return default + + if self.statistics: + sql(cache_hit) + + now = time.time() + update = 'UPDATE Cache SET %s WHERE rowid = ?' + + if update_column is not None: + sql(update % update_column.format(now=now), (rowid,)) + + if expire_time and tag: + return (value, db_expire_time, db_tag) + elif expire_time: + return (value, db_expire_time) + elif tag: + return (value, db_tag) + else: + return value + + def __getitem__(self, key): + """Return corresponding value for `key` from cache. + + :param key: key matching item + :return: corresponding value + :raises KeyError: if key is not found + + """ + value = self.get(key, default=ENOVAL, retry=True) + if value is ENOVAL: + raise KeyError(key) + return value + + def read(self, key, retry=False): + """Return file handle value corresponding to `key` from cache. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param key: key matching item + :param bool retry: retry if database timeout occurs (default False) + :return: file open for reading in binary mode + :raises KeyError: if key is not found + :raises Timeout: if database timeout occurs + + """ + handle = self.get(key, default=ENOVAL, read=True, retry=retry) + if handle is ENOVAL: + raise KeyError(key) + return handle + + def __contains__(self, key): + """Return `True` if `key` matching item is found in cache. + + :param key: key matching item + :return: True if key matching item + + """ + sql = self._sql + db_key, raw = self._disk.put(key) + select = ( + 'SELECT rowid FROM Cache' + ' WHERE key = ? AND raw = ?' + ' AND (expire_time IS NULL OR expire_time > ?)' + ) + + rows = sql(select, (db_key, raw, time.time())).fetchall() + + return bool(rows) + + def pop( + self, key, default=None, expire_time=False, tag=False, retry=False + ): # noqa: E501 + """Remove corresponding item for `key` from cache and return value. + + If `key` is missing, return `default`. + + Operation is atomic. Concurrent operations will be serialized. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param key: key for item + :param default: value to return if key is missing (default None) + :param bool expire_time: if True, return expire_time in tuple + (default False) + :param bool tag: if True, return tag in tuple (default False) + :param bool retry: retry if database timeout occurs (default False) + :return: value for item or default if key not found + :raises Timeout: if database timeout occurs + + """ + db_key, raw = self._disk.put(key) + select = ( + 'SELECT rowid, expire_time, tag, mode, filename, value' + ' FROM Cache WHERE key = ? AND raw = ?' + ' AND (expire_time IS NULL OR expire_time > ?)' + ) + + if expire_time and tag: + default = default, None, None + elif expire_time or tag: + default = default, None + + with self._transact(retry) as (sql, _): + rows = sql(select, (db_key, raw, time.time())).fetchall() + + if not rows: + return default + + ((rowid, db_expire_time, db_tag, mode, filename, db_value),) = rows + + sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) + + try: + value = self._disk.fetch(mode, filename, db_value, False) + except IOError: + # Key was deleted before we could retrieve result. + return default + finally: + if filename is not None: + self._disk.remove(filename) + + if expire_time and tag: + return value, db_expire_time, db_tag + elif expire_time: + return value, db_expire_time + elif tag: + return value, db_tag + else: + return value + + def __delitem__(self, key, retry=True): + """Delete corresponding item for `key` from cache. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default `True`). + + :param key: key matching item + :param bool retry: retry if database timeout occurs (default True) + :raises KeyError: if key is not found + :raises Timeout: if database timeout occurs + + """ + db_key, raw = self._disk.put(key) + + with self._transact(retry) as (sql, cleanup): + rows = sql( + 'SELECT rowid, filename FROM Cache' + ' WHERE key = ? AND raw = ?' + ' AND (expire_time IS NULL OR expire_time > ?)', + (db_key, raw, time.time()), + ).fetchall() + + if not rows: + raise KeyError(key) + + ((rowid, filename),) = rows + sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) + cleanup(filename) + + return True + + def delete(self, key, retry=False): + """Delete corresponding item for `key` from cache. + + Missing keys are ignored. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param key: key matching item + :param bool retry: retry if database timeout occurs (default False) + :return: True if item was deleted + :raises Timeout: if database timeout occurs + + """ + # pylint: disable=unnecessary-dunder-call + try: + return self.__delitem__(key, retry=retry) + except KeyError: + return False + + def push( + self, + value, + prefix=None, + side='back', + expire=None, + read=False, + tag=None, + retry=False, + ): + """Push `value` onto `side` of queue identified by `prefix` in cache. + + When prefix is None, integer keys are used. Otherwise, string keys are + used in the format "prefix-integer". Integer starts at 500 trillion. + + Defaults to pushing value on back of queue. Set side to 'front' to push + value on front of queue. Side must be one of 'back' or 'front'. + + Operation is atomic. Concurrent operations will be serialized. + + When `read` is `True`, `value` should be a file-like object opened + for reading in binary mode. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + See also `Cache.pull`. + + >>> cache = Cache() + >>> print(cache.push('first value')) + 500000000000000 + >>> cache.get(500000000000000) + 'first value' + >>> print(cache.push('second value')) + 500000000000001 + >>> print(cache.push('third value', side='front')) + 499999999999999 + >>> cache.push(1234, prefix='userids') + 'userids-500000000000000' + + :param value: value for item + :param str prefix: key prefix (default None, key is integer) + :param str side: either 'back' or 'front' (default 'back') + :param float expire: seconds until the key expires + (default None, no expiry) + :param bool read: read value as bytes from file (default False) + :param str tag: text to associate with key (default None) + :param bool retry: retry if database timeout occurs (default False) + :return: key for item in cache + :raises Timeout: if database timeout occurs + + """ + if prefix is None: + min_key = 0 + max_key = 999999999999999 + else: + min_key = prefix + '-000000000000000' + max_key = prefix + '-999999999999999' + + now = time.time() + raw = True + expire_time = None if expire is None else now + expire + size, mode, filename, db_value = self._disk.store(value, read) + columns = (expire_time, tag, size, mode, filename, db_value) + order = {'back': 'DESC', 'front': 'ASC'} + select = ( + 'SELECT key FROM Cache' + ' WHERE ? < key AND key < ? AND raw = ?' + ' ORDER BY key %s LIMIT 1' + ) % order[side] + + with self._transact(retry, filename) as (sql, cleanup): + rows = sql(select, (min_key, max_key, raw)).fetchall() + + if rows: + ((key,),) = rows + + if prefix is not None: + num = int(key[(key.rfind('-') + 1) :]) + else: + num = key + + if side == 'back': + num += 1 + else: + assert side == 'front' + num -= 1 + else: + num = 500000000000000 + + if prefix is not None: + db_key = '{0}-{1:015d}'.format(prefix, num) + else: + db_key = num + + self._row_insert(db_key, raw, now, columns) + self._cull(now, sql, cleanup) + + return db_key + + def pull( + self, + prefix=None, + default=(None, None), + side='front', + expire_time=False, + tag=False, + retry=False, + ): + """Pull key and value item pair from `side` of queue in cache. + + When prefix is None, integer keys are used. Otherwise, string keys are + used in the format "prefix-integer". Integer starts at 500 trillion. + + If queue is empty, return default. + + Defaults to pulling key and value item pairs from front of queue. Set + side to 'back' to pull from back of queue. Side must be one of 'front' + or 'back'. + + Operation is atomic. Concurrent operations will be serialized. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + See also `Cache.push` and `Cache.get`. + + >>> cache = Cache() + >>> cache.pull() + (None, None) + >>> for letter in 'abc': + ... print(cache.push(letter)) + 500000000000000 + 500000000000001 + 500000000000002 + >>> key, value = cache.pull() + >>> print(key) + 500000000000000 + >>> value + 'a' + >>> _, value = cache.pull(side='back') + >>> value + 'c' + >>> cache.push(1234, 'userids') + 'userids-500000000000000' + >>> _, value = cache.pull('userids') + >>> value + 1234 + + :param str prefix: key prefix (default None, key is integer) + :param default: value to return if key is missing + (default (None, None)) + :param str side: either 'front' or 'back' (default 'front') + :param bool expire_time: if True, return expire_time in tuple + (default False) + :param bool tag: if True, return tag in tuple (default False) + :param bool retry: retry if database timeout occurs (default False) + :return: key and value item pair or default if queue is empty + :raises Timeout: if database timeout occurs + + """ + # Caution: Nearly identical code exists in Cache.peek + if prefix is None: + min_key = 0 + max_key = 999999999999999 + else: + min_key = prefix + '-000000000000000' + max_key = prefix + '-999999999999999' + + order = {'front': 'ASC', 'back': 'DESC'} + select = ( + 'SELECT rowid, key, expire_time, tag, mode, filename, value' + ' FROM Cache WHERE ? < key AND key < ? AND raw = 1' + ' ORDER BY key %s LIMIT 1' + ) % order[side] + + if expire_time and tag: + default = default, None, None + elif expire_time or tag: + default = default, None + + while True: + while True: + with self._transact(retry) as (sql, cleanup): + rows = sql(select, (min_key, max_key)).fetchall() + + if not rows: + return default + + ( + (rowid, key, db_expire, db_tag, mode, name, db_value), + ) = rows + + sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) + + if db_expire is not None and db_expire < time.time(): + cleanup(name) + else: + break + + try: + value = self._disk.fetch(mode, name, db_value, False) + except IOError: + # Key was deleted before we could retrieve result. + continue + finally: + if name is not None: + self._disk.remove(name) + break + + if expire_time and tag: + return (key, value), db_expire, db_tag + elif expire_time: + return (key, value), db_expire + elif tag: + return (key, value), db_tag + else: + return key, value + + def peek( + self, + prefix=None, + default=(None, None), + side='front', + expire_time=False, + tag=False, + retry=False, + ): + """Peek at key and value item pair from `side` of queue in cache. + + When prefix is None, integer keys are used. Otherwise, string keys are + used in the format "prefix-integer". Integer starts at 500 trillion. + + If queue is empty, return default. + + Defaults to peeking at key and value item pairs from front of queue. + Set side to 'back' to pull from back of queue. Side must be one of + 'front' or 'back'. + + Expired items are deleted from cache. Operation is atomic. Concurrent + operations will be serialized. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + See also `Cache.pull` and `Cache.push`. + + >>> cache = Cache() + >>> for letter in 'abc': + ... print(cache.push(letter)) + 500000000000000 + 500000000000001 + 500000000000002 + >>> key, value = cache.peek() + >>> print(key) + 500000000000000 + >>> value + 'a' + >>> key, value = cache.peek(side='back') + >>> print(key) + 500000000000002 + >>> value + 'c' + + :param str prefix: key prefix (default None, key is integer) + :param default: value to return if key is missing + (default (None, None)) + :param str side: either 'front' or 'back' (default 'front') + :param bool expire_time: if True, return expire_time in tuple + (default False) + :param bool tag: if True, return tag in tuple (default False) + :param bool retry: retry if database timeout occurs (default False) + :return: key and value item pair or default if queue is empty + :raises Timeout: if database timeout occurs + + """ + # Caution: Nearly identical code exists in Cache.pull + if prefix is None: + min_key = 0 + max_key = 999999999999999 + else: + min_key = prefix + '-000000000000000' + max_key = prefix + '-999999999999999' + + order = {'front': 'ASC', 'back': 'DESC'} + select = ( + 'SELECT rowid, key, expire_time, tag, mode, filename, value' + ' FROM Cache WHERE ? < key AND key < ? AND raw = 1' + ' ORDER BY key %s LIMIT 1' + ) % order[side] + + if expire_time and tag: + default = default, None, None + elif expire_time or tag: + default = default, None + + while True: + while True: + with self._transact(retry) as (sql, cleanup): + rows = sql(select, (min_key, max_key)).fetchall() + + if not rows: + return default + + ( + (rowid, key, db_expire, db_tag, mode, name, db_value), + ) = rows + + if db_expire is not None and db_expire < time.time(): + sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) + cleanup(name) + else: + break + + try: + value = self._disk.fetch(mode, name, db_value, False) + except IOError: + # Key was deleted before we could retrieve result. + continue + break + + if expire_time and tag: + return (key, value), db_expire, db_tag + elif expire_time: + return (key, value), db_expire + elif tag: + return (key, value), db_tag + else: + return key, value + + def peekitem(self, last=True, expire_time=False, tag=False, retry=False): + """Peek at key and value item pair in cache based on iteration order. + + Expired items are deleted from cache. Operation is atomic. Concurrent + operations will be serialized. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + >>> cache = Cache() + >>> for num, letter in enumerate('abc'): + ... cache[letter] = num + >>> cache.peekitem() + ('c', 2) + >>> cache.peekitem(last=False) + ('a', 0) + + :param bool last: last item in iteration order (default True) + :param bool expire_time: if True, return expire_time in tuple + (default False) + :param bool tag: if True, return tag in tuple (default False) + :param bool retry: retry if database timeout occurs (default False) + :return: key and value item pair + :raises KeyError: if cache is empty + :raises Timeout: if database timeout occurs + + """ + order = ('ASC', 'DESC') + select = ( + 'SELECT rowid, key, raw, expire_time, tag, mode, filename, value' + ' FROM Cache ORDER BY rowid %s LIMIT 1' + ) % order[last] + + while True: + while True: + with self._transact(retry) as (sql, cleanup): + rows = sql(select).fetchall() + + if not rows: + raise KeyError('dictionary is empty') + + ( + ( + rowid, + db_key, + raw, + db_expire, + db_tag, + mode, + name, + db_value, + ), + ) = rows + + if db_expire is not None and db_expire < time.time(): + sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) + cleanup(name) + else: + break + + key = self._disk.get(db_key, raw) + + try: + value = self._disk.fetch(mode, name, db_value, False) + except IOError: + # Key was deleted before we could retrieve result. + continue + break + + if expire_time and tag: + return (key, value), db_expire, db_tag + elif expire_time: + return (key, value), db_expire + elif tag: + return (key, value), db_tag + else: + return key, value + + def memoize( + self, name=None, typed=False, expire=None, tag=None, ignore=() + ): + """Memoizing cache decorator. + + Decorator to wrap callable with memoizing function using cache. + Repeated calls with the same arguments will lookup result in cache and + avoid function evaluation. + + If name is set to None (default), the callable name will be determined + automatically. + + When expire is set to zero, function results will not be set in the + cache. Cache lookups still occur, however. Read + :doc:`case-study-landing-page-caching` for example usage. + + If typed is set to True, function arguments of different types will be + cached separately. For example, f(3) and f(3.0) will be treated as + distinct calls with distinct results. + + The original underlying function is accessible through the __wrapped__ + attribute. This is useful for introspection, for bypassing the cache, + or for rewrapping the function with a different cache. + + >>> from diskcache import Cache + >>> cache = Cache() + >>> @cache.memoize(expire=1, tag='fib') + ... def fibonacci(number): + ... if number == 0: + ... return 0 + ... elif number == 1: + ... return 1 + ... else: + ... return fibonacci(number - 1) + fibonacci(number - 2) + >>> print(fibonacci(100)) + 354224848179261915075 + + An additional `__cache_key__` attribute can be used to generate the + cache key used for the given arguments. + + >>> key = fibonacci.__cache_key__(100) + >>> print(cache[key]) + 354224848179261915075 + + Remember to call memoize when decorating a callable. If you forget, + then a TypeError will occur. Note the lack of parenthenses after + memoize below: + + >>> @cache.memoize + ... def test(): + ... pass + Traceback (most recent call last): + ... + TypeError: name cannot be callable + + :param cache: cache to store callable arguments and return values + :param str name: name given for callable (default None, automatic) + :param bool typed: cache different types separately (default False) + :param float expire: seconds until arguments expire + (default None, no expiry) + :param str tag: text to associate with arguments (default None) + :param set ignore: positional or keyword args to ignore (default ()) + :return: callable decorator + + """ + # Caution: Nearly identical code exists in DjangoCache.memoize + if callable(name): + raise TypeError('name cannot be callable') + + def decorator(func): + """Decorator created by memoize() for callable `func`.""" + base = (full_name(func),) if name is None else (name,) + + @ft.wraps(func) + def wrapper(*args, **kwargs): + """Wrapper for callable to cache arguments and return values.""" + key = wrapper.__cache_key__(*args, **kwargs) + result = self.get(key, default=ENOVAL, retry=True) + + if result is ENOVAL: + result = func(*args, **kwargs) + if expire is None or expire > 0: + self.set(key, result, expire, tag=tag, retry=True) + + return result + + def __cache_key__(*args, **kwargs): + """Make key for cache given function arguments.""" + return args_to_key(base, args, kwargs, typed, ignore) + + wrapper.__cache_key__ = __cache_key__ + return wrapper + + return decorator + + def check(self, fix=False, retry=False): + """Check database and file system consistency. + + Intended for use in testing and post-mortem error analysis. + + While checking the Cache table for consistency, a writer lock is held + on the database. The lock blocks other cache clients from writing to + the database. For caches with many file references, the lock may be + held for a long time. For example, local benchmarking shows that a + cache with 1,000 file references takes ~60ms to check. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param bool fix: correct inconsistencies + :param bool retry: retry if database timeout occurs (default False) + :return: list of warnings + :raises Timeout: if database timeout occurs + + """ + # pylint: disable=access-member-before-definition,W0201 + with warnings.catch_warnings(record=True) as warns: + sql = self._sql + + # Check integrity of database. + + rows = sql('PRAGMA integrity_check').fetchall() + + if len(rows) != 1 or rows[0][0] != 'ok': + for (message,) in rows: + warnings.warn(message) + + if fix: + sql('VACUUM') + + with self._transact(retry) as (sql, _): + + # Check Cache.filename against file system. + + filenames = set() + select = ( + 'SELECT rowid, size, filename FROM Cache' + ' WHERE filename IS NOT NULL' + ) + + rows = sql(select).fetchall() + + for rowid, size, filename in rows: + full_path = op.join(self._directory, filename) + filenames.add(full_path) + + if op.exists(full_path): + real_size = op.getsize(full_path) + + if size != real_size: + message = 'wrong file size: %s, %d != %d' + args = full_path, real_size, size + warnings.warn(message % args) + + if fix: + sql( + 'UPDATE Cache SET size = ?' + ' WHERE rowid = ?', + (real_size, rowid), + ) + + continue + + warnings.warn('file not found: %s' % full_path) + + if fix: + sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) + + # Check file system against Cache.filename. + + for dirpath, _, files in os.walk(self._directory): + paths = [op.join(dirpath, filename) for filename in files] + error = set(paths) - filenames + + for full_path in error: + if DBNAME in full_path: + continue + + message = 'unknown file: %s' % full_path + warnings.warn(message, UnknownFileWarning) + + if fix: + os.remove(full_path) + + # Check for empty directories. + + for dirpath, dirs, files in os.walk(self._directory): + if not (dirs or files): + message = 'empty directory: %s' % dirpath + warnings.warn(message, EmptyDirWarning) + + if fix: + os.rmdir(dirpath) + + # Check Settings.count against count of Cache rows. + + self.reset('count') + ((count,),) = sql('SELECT COUNT(key) FROM Cache').fetchall() + + if self.count != count: + message = 'Settings.count != COUNT(Cache.key); %d != %d' + warnings.warn(message % (self.count, count)) + + if fix: + sql( + 'UPDATE Settings SET value = ? WHERE key = ?', + (count, 'count'), + ) + + # Check Settings.size against sum of Cache.size column. + + self.reset('size') + select_size = 'SELECT COALESCE(SUM(size), 0) FROM Cache' + ((size,),) = sql(select_size).fetchall() + + if self.size != size: + message = 'Settings.size != SUM(Cache.size); %d != %d' + warnings.warn(message % (self.size, size)) + + if fix: + sql( + 'UPDATE Settings SET value = ? WHERE key =?', + (size, 'size'), + ) + + return warns + + def create_tag_index(self): + """Create tag index on cache database. + + It is better to initialize cache with `tag_index=True` than use this. + + :raises Timeout: if database timeout occurs + + """ + sql = self._sql + sql('CREATE INDEX IF NOT EXISTS Cache_tag_rowid ON Cache(tag, rowid)') + self.reset('tag_index', 1) + + def drop_tag_index(self): + """Drop tag index on cache database. + + :raises Timeout: if database timeout occurs + + """ + sql = self._sql + sql('DROP INDEX IF EXISTS Cache_tag_rowid') + self.reset('tag_index', 0) + + def evict(self, tag, retry=False): + """Remove items with matching `tag` from cache. + + Removing items is an iterative process. In each iteration, a subset of + items is removed. Concurrent writes may occur between iterations. + + If a :exc:`Timeout` occurs, the first element of the exception's + `args` attribute will be the number of items removed before the + exception occurred. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param str tag: tag identifying items + :param bool retry: retry if database timeout occurs (default False) + :return: count of rows removed + :raises Timeout: if database timeout occurs + + """ + select = ( + 'SELECT rowid, filename FROM Cache' + ' WHERE tag = ? AND rowid > ?' + ' ORDER BY rowid LIMIT ?' + ) + args = [tag, 0, 100] + return self._select_delete(select, args, arg_index=1, retry=retry) + + def expire(self, now=None, retry=False): + """Remove expired items from cache. + + Removing items is an iterative process. In each iteration, a subset of + items is removed. Concurrent writes may occur between iterations. + + If a :exc:`Timeout` occurs, the first element of the exception's + `args` attribute will be the number of items removed before the + exception occurred. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param float now: current time (default None, ``time.time()`` used) + :param bool retry: retry if database timeout occurs (default False) + :return: count of items removed + :raises Timeout: if database timeout occurs + + """ + select = ( + 'SELECT rowid, expire_time, filename FROM Cache' + ' WHERE ? < expire_time AND expire_time < ?' + ' ORDER BY expire_time LIMIT ?' + ) + args = [0, now or time.time(), 100] + return self._select_delete(select, args, row_index=1, retry=retry) + + def cull(self, retry=False): + """Cull items from cache until volume is less than size limit. + + Removing items is an iterative process. In each iteration, a subset of + items is removed. Concurrent writes may occur between iterations. + + If a :exc:`Timeout` occurs, the first element of the exception's + `args` attribute will be the number of items removed before the + exception occurred. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param bool retry: retry if database timeout occurs (default False) + :return: count of items removed + :raises Timeout: if database timeout occurs + + """ + now = time.time() + + # Remove expired items. + + count = self.expire(now) + + # Remove items by policy. + + select_policy = EVICTION_POLICY[self.eviction_policy]['cull'] + + if select_policy is None: + return 0 + + select_filename = select_policy.format(fields='filename', now=now) + + try: + while self.volume() > self.size_limit: + with self._transact(retry) as (sql, cleanup): + rows = sql(select_filename, (10,)).fetchall() + + if not rows: + break + + count += len(rows) + delete = ( + 'DELETE FROM Cache WHERE rowid IN (%s)' + % select_policy.format(fields='rowid', now=now) + ) + sql(delete, (10,)) + + for (filename,) in rows: + cleanup(filename) + except Timeout: + raise Timeout(count) from None + + return count + + def clear(self, retry=False): + """Remove all items from cache. + + Removing items is an iterative process. In each iteration, a subset of + items is removed. Concurrent writes may occur between iterations. + + If a :exc:`Timeout` occurs, the first element of the exception's + `args` attribute will be the number of items removed before the + exception occurred. + + Raises :exc:`Timeout` error when database timeout occurs and `retry` is + `False` (default). + + :param bool retry: retry if database timeout occurs (default False) + :return: count of rows removed + :raises Timeout: if database timeout occurs + + """ + select = ( + 'SELECT rowid, filename FROM Cache' + ' WHERE rowid > ?' + ' ORDER BY rowid LIMIT ?' + ) + args = [0, 100] + return self._select_delete(select, args, retry=retry) + + def _select_delete( + self, select, args, row_index=0, arg_index=0, retry=False + ): + count = 0 + delete = 'DELETE FROM Cache WHERE rowid IN (%s)' + + try: + while True: + with self._transact(retry) as (sql, cleanup): + rows = sql(select, args).fetchall() + + if not rows: + break + + count += len(rows) + sql(delete % ','.join(str(row[0]) for row in rows)) + + for row in rows: + args[arg_index] = row[row_index] + cleanup(row[-1]) + + except Timeout: + raise Timeout(count) from None + + return count + + def iterkeys(self, reverse=False): + """Iterate Cache keys in database sort order. + + >>> cache = Cache() + >>> for key in [4, 1, 3, 0, 2]: + ... cache[key] = key + >>> list(cache.iterkeys()) + [0, 1, 2, 3, 4] + >>> list(cache.iterkeys(reverse=True)) + [4, 3, 2, 1, 0] + + :param bool reverse: reverse sort order (default False) + :return: iterator of Cache keys + + """ + sql = self._sql + limit = 100 + _disk_get = self._disk.get + + if reverse: + select = ( + 'SELECT key, raw FROM Cache' + ' ORDER BY key DESC, raw DESC LIMIT 1' + ) + iterate = ( + 'SELECT key, raw FROM Cache' + ' WHERE key = ? AND raw < ? OR key < ?' + ' ORDER BY key DESC, raw DESC LIMIT ?' + ) + else: + select = ( + 'SELECT key, raw FROM Cache' + ' ORDER BY key ASC, raw ASC LIMIT 1' + ) + iterate = ( + 'SELECT key, raw FROM Cache' + ' WHERE key = ? AND raw > ? OR key > ?' + ' ORDER BY key ASC, raw ASC LIMIT ?' + ) + + row = sql(select).fetchall() + + if row: + ((key, raw),) = row + else: + return + + yield _disk_get(key, raw) + + while True: + rows = sql(iterate, (key, raw, key, limit)).fetchall() + + if not rows: + break + + for key, raw in rows: + yield _disk_get(key, raw) + + def _iter(self, ascending=True): + sql = self._sql + rows = sql('SELECT MAX(rowid) FROM Cache').fetchall() + ((max_rowid,),) = rows + yield # Signal ready. + + if max_rowid is None: + return + + bound = max_rowid + 1 + limit = 100 + _disk_get = self._disk.get + rowid = 0 if ascending else bound + select = ( + 'SELECT rowid, key, raw FROM Cache' + ' WHERE ? < rowid AND rowid < ?' + ' ORDER BY rowid %s LIMIT ?' + ) % ('ASC' if ascending else 'DESC') + + while True: + if ascending: + args = (rowid, bound, limit) + else: + args = (0, rowid, limit) + + rows = sql(select, args).fetchall() + + if not rows: + break + + for rowid, key, raw in rows: + yield _disk_get(key, raw) + + def __iter__(self): + """Iterate keys in cache including expired items.""" + iterator = self._iter() + next(iterator) + return iterator + + def __reversed__(self): + """Reverse iterate keys in cache including expired items.""" + iterator = self._iter(ascending=False) + next(iterator) + return iterator + + def stats(self, enable=True, reset=False): + """Return cache statistics hits and misses. + + :param bool enable: enable collecting statistics (default True) + :param bool reset: reset hits and misses to 0 (default False) + :return: (hits, misses) + + """ + # pylint: disable=E0203,W0201 + result = (self.reset('hits'), self.reset('misses')) + + if reset: + self.reset('hits', 0) + self.reset('misses', 0) + + self.reset('statistics', enable) + + return result + + def volume(self): + """Return estimated total size of cache on disk. + + :return: size in bytes + + """ + ((page_count,),) = self._sql('PRAGMA page_count').fetchall() + total_size = self._page_size * page_count + self.reset('size') + return total_size + + def close(self): + """Close database connection.""" + con = getattr(self._local, 'con', None) + + if con is None: + return + + con.close() + + try: + delattr(self._local, 'con') + except AttributeError: + pass + + def __enter__(self): + # Create connection in thread. + # pylint: disable=unused-variable + connection = self._con # noqa + return self + + def __exit__(self, *exception): + self.close() + + def __len__(self): + """Count of items in cache including expired items.""" + return self.reset('count') + + def __getstate__(self): + return (self.directory, self.timeout, type(self.disk)) + + def __setstate__(self, state): + self.__init__(*state) + + def reset(self, key, value=ENOVAL, update=True): + """Reset `key` and `value` item from Settings table. + + Use `reset` to update the value of Cache settings correctly. Cache + settings are stored in the Settings table of the SQLite database. If + `update` is ``False`` then no attempt is made to update the database. + + If `value` is not given, it is reloaded from the Settings + table. Otherwise, the Settings table is updated. + + Settings with the ``disk_`` prefix correspond to Disk + attributes. Updating the value will change the unprefixed attribute on + the associated Disk instance. + + Settings with the ``sqlite_`` prefix correspond to SQLite + pragmas. Updating the value will execute the corresponding PRAGMA + statement. + + SQLite PRAGMA statements may be executed before the Settings table + exists in the database by setting `update` to ``False``. + + :param str key: Settings key for item + :param value: value for item (optional) + :param bool update: update database Settings table (default True) + :return: updated value for item + :raises Timeout: if database timeout occurs + + """ + sql = self._sql + sql_retry = self._sql_retry + + if value is ENOVAL: + select = 'SELECT value FROM Settings WHERE key = ?' + ((value,),) = sql_retry(select, (key,)).fetchall() + setattr(self, key, value) + return value + + if update: + statement = 'UPDATE Settings SET value = ? WHERE key = ?' + sql_retry(statement, (value, key)) + + if key.startswith('sqlite_'): + pragma = key[7:] + + # 2016-02-17 GrantJ - PRAGMA and isolation_level=None + # don't always play nicely together. Retry setting the + # PRAGMA. I think some PRAGMA statements expect to + # immediately take an EXCLUSIVE lock on the database. I + # can't find any documentation for this but without the + # retry, stress will intermittently fail with multiple + # processes. + + # 2018-11-05 GrantJ - Avoid setting pragma values that + # are already set. Pragma settings like auto_vacuum and + # journal_mode can take a long time or may not work after + # tables have been created. + + start = time.time() + while True: + try: + try: + ((old_value,),) = sql( + 'PRAGMA %s' % (pragma) + ).fetchall() + update = old_value != value + except ValueError: + update = True + if update: + sql('PRAGMA %s = %s' % (pragma, value)).fetchall() + break + except sqlite3.OperationalError as exc: + if str(exc) != 'database is locked': + raise + diff = time.time() - start + if diff > 60: + raise + time.sleep(0.001) + elif key.startswith('disk_'): + attr = key[5:] + setattr(self._disk, attr, value) + + setattr(self, key, value) + return value diff --git a/lib/python3.1/site-packages/diskcache/djangocache.py b/lib/python3.1/site-packages/diskcache/djangocache.py new file mode 100644 index 0000000000000000000000000000000000000000..5dc8ce2c5f98d6cceeff7fee5888b680f1e3b39a --- /dev/null +++ b/lib/python3.1/site-packages/diskcache/djangocache.py @@ -0,0 +1,456 @@ +"""Django-compatible disk and file backed cache.""" + +from functools import wraps + +from django.core.cache.backends.base import BaseCache + +try: + from django.core.cache.backends.base import DEFAULT_TIMEOUT +except ImportError: # pragma: no cover + # For older versions of Django simply use 300 seconds. + DEFAULT_TIMEOUT = 300 + +from .core import ENOVAL, args_to_key, full_name +from .fanout import FanoutCache + + +class DjangoCache(BaseCache): + """Django-compatible disk and file backed cache.""" + + def __init__(self, directory, params): + """Initialize DjangoCache instance. + + :param str directory: cache directory + :param dict params: cache parameters + + """ + super().__init__(params) + shards = params.get('SHARDS', 8) + timeout = params.get('DATABASE_TIMEOUT', 0.010) + options = params.get('OPTIONS', {}) + self._cache = FanoutCache(directory, shards, timeout, **options) + + @property + def directory(self): + """Cache directory.""" + return self._cache.directory + + def cache(self, name): + """Return Cache with given `name` in subdirectory. + + :param str name: subdirectory name for Cache + :return: Cache with given name + + """ + return self._cache.cache(name) + + def deque(self, name, maxlen=None): + """Return Deque with given `name` in subdirectory. + + :param str name: subdirectory name for Deque + :param maxlen: max length (default None, no max) + :return: Deque with given name + + """ + return self._cache.deque(name, maxlen=maxlen) + + def index(self, name): + """Return Index with given `name` in subdirectory. + + :param str name: subdirectory name for Index + :return: Index with given name + + """ + return self._cache.index(name) + + def add( + self, + key, + value, + timeout=DEFAULT_TIMEOUT, + version=None, + read=False, + tag=None, + retry=True, + ): + """Set a value in the cache if the key does not already exist. If + timeout is given, that timeout will be used for the key; otherwise the + default cache timeout will be used. + + Return True if the value was stored, False otherwise. + + :param key: key for item + :param value: value for item + :param float timeout: seconds until the item expires + (default 300 seconds) + :param int version: key version number (default None, cache parameter) + :param bool read: read value as bytes from file (default False) + :param str tag: text to associate with key (default None) + :param bool retry: retry if database timeout occurs (default True) + :return: True if item was added + + """ + # pylint: disable=arguments-differ + key = self.make_key(key, version=version) + timeout = self.get_backend_timeout(timeout=timeout) + return self._cache.add(key, value, timeout, read, tag, retry) + + def get( + self, + key, + default=None, + version=None, + read=False, + expire_time=False, + tag=False, + retry=False, + ): + """Fetch a given key from the cache. If the key does not exist, return + default, which itself defaults to None. + + :param key: key for item + :param default: return value if key is missing (default None) + :param int version: key version number (default None, cache parameter) + :param bool read: if True, return file handle to value + (default False) + :param float expire_time: if True, return expire_time in tuple + (default False) + :param tag: if True, return tag in tuple (default False) + :param bool retry: retry if database timeout occurs (default False) + :return: value for item if key is found else default + + """ + # pylint: disable=arguments-differ + key = self.make_key(key, version=version) + return self._cache.get(key, default, read, expire_time, tag, retry) + + def read(self, key, version=None): + """Return file handle corresponding to `key` from Cache. + + :param key: Python key to retrieve + :param int version: key version number (default None, cache parameter) + :return: file open for reading in binary mode + :raises KeyError: if key is not found + + """ + key = self.make_key(key, version=version) + return self._cache.read(key) + + def set( + self, + key, + value, + timeout=DEFAULT_TIMEOUT, + version=None, + read=False, + tag=None, + retry=True, + ): + """Set a value in the cache. If timeout is given, that timeout will be + used for the key; otherwise the default cache timeout will be used. + + :param key: key for item + :param value: value for item + :param float timeout: seconds until the item expires + (default 300 seconds) + :param int version: key version number (default None, cache parameter) + :param bool read: read value as bytes from file (default False) + :param str tag: text to associate with key (default None) + :param bool retry: retry if database timeout occurs (default True) + :return: True if item was set + + """ + # pylint: disable=arguments-differ + key = self.make_key(key, version=version) + timeout = self.get_backend_timeout(timeout=timeout) + return self._cache.set(key, value, timeout, read, tag, retry) + + def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None, retry=True): + """Touch a key in the cache. If timeout is given, that timeout will be + used for the key; otherwise the default cache timeout will be used. + + :param key: key for item + :param float timeout: seconds until the item expires + (default 300 seconds) + :param int version: key version number (default None, cache parameter) + :param bool retry: retry if database timeout occurs (default True) + :return: True if key was touched + + """ + # pylint: disable=arguments-differ + key = self.make_key(key, version=version) + timeout = self.get_backend_timeout(timeout=timeout) + return self._cache.touch(key, timeout, retry) + + def pop( + self, + key, + default=None, + version=None, + expire_time=False, + tag=False, + retry=True, + ): + """Remove corresponding item for `key` from cache and return value. + + If `key` is missing, return `default`. + + Operation is atomic. Concurrent operations will be serialized. + + :param key: key for item + :param default: return value if key is missing (default None) + :param int version: key version number (default None, cache parameter) + :param float expire_time: if True, return expire_time in tuple + (default False) + :param tag: if True, return tag in tuple (default False) + :param bool retry: retry if database timeout occurs (default True) + :return: value for item if key is found else default + + """ + key = self.make_key(key, version=version) + return self._cache.pop(key, default, expire_time, tag, retry) + + def delete(self, key, version=None, retry=True): + """Delete a key from the cache, failing silently. + + :param key: key for item + :param int version: key version number (default None, cache parameter) + :param bool retry: retry if database timeout occurs (default True) + :return: True if item was deleted + + """ + # pylint: disable=arguments-differ + key = self.make_key(key, version=version) + return self._cache.delete(key, retry) + + def incr(self, key, delta=1, version=None, default=None, retry=True): + """Increment value by delta for item with key. + + If key is missing and default is None then raise KeyError. Else if key + is missing and default is not None then use default for value. + + Operation is atomic. All concurrent increment operations will be + counted individually. + + Assumes value may be stored in a SQLite column. Most builds that target + machines with 64-bit pointer widths will support 64-bit signed + integers. + + :param key: key for item + :param int delta: amount to increment (default 1) + :param int version: key version number (default None, cache parameter) + :param int default: value if key is missing (default None) + :param bool retry: retry if database timeout occurs (default True) + :return: new value for item on success else None + :raises ValueError: if key is not found and default is None + + """ + # pylint: disable=arguments-differ + key = self.make_key(key, version=version) + try: + return self._cache.incr(key, delta, default, retry) + except KeyError: + raise ValueError("Key '%s' not found" % key) from None + + def decr(self, key, delta=1, version=None, default=None, retry=True): + """Decrement value by delta for item with key. + + If key is missing and default is None then raise KeyError. Else if key + is missing and default is not None then use default for value. + + Operation is atomic. All concurrent decrement operations will be + counted individually. + + Unlike Memcached, negative values are supported. Value may be + decremented below zero. + + Assumes value may be stored in a SQLite column. Most builds that target + machines with 64-bit pointer widths will support 64-bit signed + integers. + + :param key: key for item + :param int delta: amount to decrement (default 1) + :param int version: key version number (default None, cache parameter) + :param int default: value if key is missing (default None) + :param bool retry: retry if database timeout occurs (default True) + :return: new value for item on success else None + :raises ValueError: if key is not found and default is None + + """ + # pylint: disable=arguments-differ + return self.incr(key, -delta, version, default, retry) + + def has_key(self, key, version=None): + """Returns True if the key is in the cache and has not expired. + + :param key: key for item + :param int version: key version number (default None, cache parameter) + :return: True if key is found + + """ + key = self.make_key(key, version=version) + return key in self._cache + + def expire(self): + """Remove expired items from cache. + + :return: count of items removed + + """ + return self._cache.expire() + + def stats(self, enable=True, reset=False): + """Return cache statistics hits and misses. + + :param bool enable: enable collecting statistics (default True) + :param bool reset: reset hits and misses to 0 (default False) + :return: (hits, misses) + + """ + return self._cache.stats(enable=enable, reset=reset) + + def create_tag_index(self): + """Create tag index on cache database. + + Better to initialize cache with `tag_index=True` than use this. + + :raises Timeout: if database timeout occurs + + """ + self._cache.create_tag_index() + + def drop_tag_index(self): + """Drop tag index on cache database. + + :raises Timeout: if database timeout occurs + + """ + self._cache.drop_tag_index() + + def evict(self, tag): + """Remove items with matching `tag` from cache. + + :param str tag: tag identifying items + :return: count of items removed + + """ + return self._cache.evict(tag) + + def cull(self): + """Cull items from cache until volume is less than size limit. + + :return: count of items removed + + """ + return self._cache.cull() + + def clear(self): + """Remove *all* values from the cache at once.""" + return self._cache.clear() + + def close(self, **kwargs): + """Close the cache connection.""" + # pylint: disable=unused-argument + self._cache.close() + + def get_backend_timeout(self, timeout=DEFAULT_TIMEOUT): + """Return seconds to expiration. + + :param float timeout: seconds until the item expires + (default 300 seconds) + + """ + if timeout == DEFAULT_TIMEOUT: + timeout = self.default_timeout + elif timeout == 0: + # ticket 21147 - avoid time.time() related precision issues + timeout = -1 + return None if timeout is None else timeout + + def memoize( + self, + name=None, + timeout=DEFAULT_TIMEOUT, + version=None, + typed=False, + tag=None, + ignore=(), + ): + """Memoizing cache decorator. + + Decorator to wrap callable with memoizing function using cache. + Repeated calls with the same arguments will lookup result in cache and + avoid function evaluation. + + If name is set to None (default), the callable name will be determined + automatically. + + When timeout is set to zero, function results will not be set in the + cache. Cache lookups still occur, however. Read + :doc:`case-study-landing-page-caching` for example usage. + + If typed is set to True, function arguments of different types will be + cached separately. For example, f(3) and f(3.0) will be treated as + distinct calls with distinct results. + + The original underlying function is accessible through the __wrapped__ + attribute. This is useful for introspection, for bypassing the cache, + or for rewrapping the function with a different cache. + + An additional `__cache_key__` attribute can be used to generate the + cache key used for the given arguments. + + Remember to call memoize when decorating a callable. If you forget, + then a TypeError will occur. + + :param str name: name given for callable (default None, automatic) + :param float timeout: seconds until the item expires + (default 300 seconds) + :param int version: key version number (default None, cache parameter) + :param bool typed: cache different types separately (default False) + :param str tag: text to associate with arguments (default None) + :param set ignore: positional or keyword args to ignore (default ()) + :return: callable decorator + + """ + # Caution: Nearly identical code exists in Cache.memoize + if callable(name): + raise TypeError('name cannot be callable') + + def decorator(func): + """Decorator created by memoize() for callable `func`.""" + base = (full_name(func),) if name is None else (name,) + + @wraps(func) + def wrapper(*args, **kwargs): + """Wrapper for callable to cache arguments and return values.""" + key = wrapper.__cache_key__(*args, **kwargs) + result = self.get(key, ENOVAL, version, retry=True) + + if result is ENOVAL: + result = func(*args, **kwargs) + valid_timeout = ( + timeout is None + or timeout == DEFAULT_TIMEOUT + or timeout > 0 + ) + if valid_timeout: + self.set( + key, + result, + timeout, + version, + tag=tag, + retry=True, + ) + + return result + + def __cache_key__(*args, **kwargs): + """Make key for cache given function arguments.""" + return args_to_key(base, args, kwargs, typed, ignore) + + wrapper.__cache_key__ = __cache_key__ + return wrapper + + return decorator diff --git a/lib/python3.1/site-packages/diskcache/fanout.py b/lib/python3.1/site-packages/diskcache/fanout.py new file mode 100644 index 0000000000000000000000000000000000000000..9822ee4e576b4d97d5e0209862f0d3a93fde5b87 --- /dev/null +++ b/lib/python3.1/site-packages/diskcache/fanout.py @@ -0,0 +1,687 @@ +"""Fanout cache automatically shards keys and values.""" + +import contextlib as cl +import functools +import itertools as it +import operator +import os.path as op +import sqlite3 +import tempfile +import time + +from .core import DEFAULT_SETTINGS, ENOVAL, Cache, Disk, Timeout +from .persistent import Deque, Index + + +class FanoutCache: + """Cache that shards keys and values.""" + + def __init__( + self, directory=None, shards=8, timeout=0.010, disk=Disk, **settings + ): + """Initialize cache instance. + + :param str directory: cache directory + :param int shards: number of shards to distribute writes + :param float timeout: SQLite connection timeout + :param disk: `Disk` instance for serialization + :param settings: any of `DEFAULT_SETTINGS` + + """ + if directory is None: + directory = tempfile.mkdtemp(prefix='diskcache-') + directory = str(directory) + directory = op.expanduser(directory) + directory = op.expandvars(directory) + + default_size_limit = DEFAULT_SETTINGS['size_limit'] + size_limit = settings.pop('size_limit', default_size_limit) / shards + + self._count = shards + self._directory = directory + self._disk = disk + self._shards = tuple( + Cache( + directory=op.join(directory, '%03d' % num), + timeout=timeout, + disk=disk, + size_limit=size_limit, + **settings, + ) + for num in range(shards) + ) + self._hash = self._shards[0].disk.hash + self._caches = {} + self._deques = {} + self._indexes = {} + + @property + def directory(self): + """Cache directory.""" + return self._directory + + def __getattr__(self, name): + safe_names = {'timeout', 'disk'} + valid_name = name in DEFAULT_SETTINGS or name in safe_names + assert valid_name, 'cannot access {} in cache shard'.format(name) + return getattr(self._shards[0], name) + + @cl.contextmanager + def transact(self, retry=True): + """Context manager to perform a transaction by locking the cache. + + While the cache is locked, no other write operation is permitted. + Transactions should therefore be as short as possible. Read and write + operations performed in a transaction are atomic. Read operations may + occur concurrent to a transaction. + + Transactions may be nested and may not be shared between threads. + + Blocks until transactions are held on all cache shards by retrying as + necessary. + + >>> cache = FanoutCache() + >>> with cache.transact(): # Atomically increment two keys. + ... _ = cache.incr('total', 123.4) + ... _ = cache.incr('count', 1) + >>> with cache.transact(): # Atomically calculate average. + ... average = cache['total'] / cache['count'] + >>> average + 123.4 + + :return: context manager for use in `with` statement + + """ + assert retry, 'retry must be True in FanoutCache' + with cl.ExitStack() as stack: + for shard in self._shards: + shard_transaction = shard.transact(retry=True) + stack.enter_context(shard_transaction) + yield + + def set(self, key, value, expire=None, read=False, tag=None, retry=False): + """Set `key` and `value` item in cache. + + When `read` is `True`, `value` should be a file-like object opened + for reading in binary mode. + + If database timeout occurs then fails silently unless `retry` is set to + `True` (default `False`). + + :param key: key for item + :param value: value for item + :param float expire: seconds until the key expires + (default None, no expiry) + :param bool read: read value as raw bytes from file (default False) + :param str tag: text to associate with key (default None) + :param bool retry: retry if database timeout occurs (default False) + :return: True if item was set + + """ + index = self._hash(key) % self._count + shard = self._shards[index] + try: + return shard.set(key, value, expire, read, tag, retry) + except Timeout: + return False + + def __setitem__(self, key, value): + """Set `key` and `value` item in cache. + + Calls :func:`FanoutCache.set` internally with `retry` set to `True`. + + :param key: key for item + :param value: value for item + + """ + index = self._hash(key) % self._count + shard = self._shards[index] + shard[key] = value + + def touch(self, key, expire=None, retry=False): + """Touch `key` in cache and update `expire` time. + + If database timeout occurs then fails silently unless `retry` is set to + `True` (default `False`). + + :param key: key for item + :param float expire: seconds until the key expires + (default None, no expiry) + :param bool retry: retry if database timeout occurs (default False) + :return: True if key was touched + + """ + index = self._hash(key) % self._count + shard = self._shards[index] + try: + return shard.touch(key, expire, retry) + except Timeout: + return False + + def add(self, key, value, expire=None, read=False, tag=None, retry=False): + """Add `key` and `value` item to cache. + + Similar to `set`, but only add to cache if key not present. + + This operation is atomic. Only one concurrent add operation for given + key from separate threads or processes will succeed. + + When `read` is `True`, `value` should be a file-like object opened + for reading in binary mode. + + If database timeout occurs then fails silently unless `retry` is set to + `True` (default `False`). + + :param key: key for item + :param value: value for item + :param float expire: seconds until the key expires + (default None, no expiry) + :param bool read: read value as bytes from file (default False) + :param str tag: text to associate with key (default None) + :param bool retry: retry if database timeout occurs (default False) + :return: True if item was added + + """ + index = self._hash(key) % self._count + shard = self._shards[index] + try: + return shard.add(key, value, expire, read, tag, retry) + except Timeout: + return False + + def incr(self, key, delta=1, default=0, retry=False): + """Increment value by delta for item with key. + + If key is missing and default is None then raise KeyError. Else if key + is missing and default is not None then use default for value. + + Operation is atomic. All concurrent increment operations will be + counted individually. + + Assumes value may be stored in a SQLite column. Most builds that target + machines with 64-bit pointer widths will support 64-bit signed + integers. + + If database timeout occurs then fails silently unless `retry` is set to + `True` (default `False`). + + :param key: key for item + :param int delta: amount to increment (default 1) + :param int default: value if key is missing (default 0) + :param bool retry: retry if database timeout occurs (default False) + :return: new value for item on success else None + :raises KeyError: if key is not found and default is None + + """ + index = self._hash(key) % self._count + shard = self._shards[index] + try: + return shard.incr(key, delta, default, retry) + except Timeout: + return None + + def decr(self, key, delta=1, default=0, retry=False): + """Decrement value by delta for item with key. + + If key is missing and default is None then raise KeyError. Else if key + is missing and default is not None then use default for value. + + Operation is atomic. All concurrent decrement operations will be + counted individually. + + Unlike Memcached, negative values are supported. Value may be + decremented below zero. + + Assumes value may be stored in a SQLite column. Most builds that target + machines with 64-bit pointer widths will support 64-bit signed + integers. + + If database timeout occurs then fails silently unless `retry` is set to + `True` (default `False`). + + :param key: key for item + :param int delta: amount to decrement (default 1) + :param int default: value if key is missing (default 0) + :param bool retry: retry if database timeout occurs (default False) + :return: new value for item on success else None + :raises KeyError: if key is not found and default is None + + """ + index = self._hash(key) % self._count + shard = self._shards[index] + try: + return shard.decr(key, delta, default, retry) + except Timeout: + return None + + def get( + self, + key, + default=None, + read=False, + expire_time=False, + tag=False, + retry=False, + ): + """Retrieve value from cache. If `key` is missing, return `default`. + + If database timeout occurs then returns `default` unless `retry` is set + to `True` (default `False`). + + :param key: key for item + :param default: return value if key is missing (default None) + :param bool read: if True, return file handle to value + (default False) + :param float expire_time: if True, return expire_time in tuple + (default False) + :param tag: if True, return tag in tuple (default False) + :param bool retry: retry if database timeout occurs (default False) + :return: value for item if key is found else default + + """ + index = self._hash(key) % self._count + shard = self._shards[index] + try: + return shard.get(key, default, read, expire_time, tag, retry) + except (Timeout, sqlite3.OperationalError): + return default + + def __getitem__(self, key): + """Return corresponding value for `key` from cache. + + Calls :func:`FanoutCache.get` internally with `retry` set to `True`. + + :param key: key for item + :return: value for item + :raises KeyError: if key is not found + + """ + index = self._hash(key) % self._count + shard = self._shards[index] + return shard[key] + + def read(self, key): + """Return file handle corresponding to `key` from cache. + + :param key: key for item + :return: file open for reading in binary mode + :raises KeyError: if key is not found + + """ + handle = self.get(key, default=ENOVAL, read=True, retry=True) + if handle is ENOVAL: + raise KeyError(key) + return handle + + def __contains__(self, key): + """Return `True` if `key` matching item is found in cache. + + :param key: key for item + :return: True if key is found + + """ + index = self._hash(key) % self._count + shard = self._shards[index] + return key in shard + + def pop( + self, key, default=None, expire_time=False, tag=False, retry=False + ): # noqa: E501 + """Remove corresponding item for `key` from cache and return value. + + If `key` is missing, return `default`. + + Operation is atomic. Concurrent operations will be serialized. + + If database timeout occurs then fails silently unless `retry` is set to + `True` (default `False`). + + :param key: key for item + :param default: return value if key is missing (default None) + :param float expire_time: if True, return expire_time in tuple + (default False) + :param tag: if True, return tag in tuple (default False) + :param bool retry: retry if database timeout occurs (default False) + :return: value for item if key is found else default + + """ + index = self._hash(key) % self._count + shard = self._shards[index] + try: + return shard.pop(key, default, expire_time, tag, retry) + except Timeout: + return default + + def delete(self, key, retry=False): + """Delete corresponding item for `key` from cache. + + Missing keys are ignored. + + If database timeout occurs then fails silently unless `retry` is set to + `True` (default `False`). + + :param key: key for item + :param bool retry: retry if database timeout occurs (default False) + :return: True if item was deleted + + """ + index = self._hash(key) % self._count + shard = self._shards[index] + try: + return shard.delete(key, retry) + except Timeout: + return False + + def __delitem__(self, key): + """Delete corresponding item for `key` from cache. + + Calls :func:`FanoutCache.delete` internally with `retry` set to `True`. + + :param key: key for item + :raises KeyError: if key is not found + + """ + index = self._hash(key) % self._count + shard = self._shards[index] + del shard[key] + + def check(self, fix=False, retry=False): + """Check database and file system consistency. + + Intended for use in testing and post-mortem error analysis. + + While checking the cache table for consistency, a writer lock is held + on the database. The lock blocks other cache clients from writing to + the database. For caches with many file references, the lock may be + held for a long time. For example, local benchmarking shows that a + cache with 1,000 file references takes ~60ms to check. + + If database timeout occurs then fails silently unless `retry` is set to + `True` (default `False`). + + :param bool fix: correct inconsistencies + :param bool retry: retry if database timeout occurs (default False) + :return: list of warnings + :raises Timeout: if database timeout occurs + + """ + warnings = (shard.check(fix, retry) for shard in self._shards) + return functools.reduce(operator.iadd, warnings, []) + + def expire(self, retry=False): + """Remove expired items from cache. + + If database timeout occurs then fails silently unless `retry` is set to + `True` (default `False`). + + :param bool retry: retry if database timeout occurs (default False) + :return: count of items removed + + """ + return self._remove('expire', args=(time.time(),), retry=retry) + + def create_tag_index(self): + """Create tag index on cache database. + + Better to initialize cache with `tag_index=True` than use this. + + :raises Timeout: if database timeout occurs + + """ + for shard in self._shards: + shard.create_tag_index() + + def drop_tag_index(self): + """Drop tag index on cache database. + + :raises Timeout: if database timeout occurs + + """ + for shard in self._shards: + shard.drop_tag_index() + + def evict(self, tag, retry=False): + """Remove items with matching `tag` from cache. + + If database timeout occurs then fails silently unless `retry` is set to + `True` (default `False`). + + :param str tag: tag identifying items + :param bool retry: retry if database timeout occurs (default False) + :return: count of items removed + + """ + return self._remove('evict', args=(tag,), retry=retry) + + def cull(self, retry=False): + """Cull items from cache until volume is less than size limit. + + If database timeout occurs then fails silently unless `retry` is set to + `True` (default `False`). + + :param bool retry: retry if database timeout occurs (default False) + :return: count of items removed + + """ + return self._remove('cull', retry=retry) + + def clear(self, retry=False): + """Remove all items from cache. + + If database timeout occurs then fails silently unless `retry` is set to + `True` (default `False`). + + :param bool retry: retry if database timeout occurs (default False) + :return: count of items removed + + """ + return self._remove('clear', retry=retry) + + def _remove(self, name, args=(), retry=False): + total = 0 + for shard in self._shards: + method = getattr(shard, name) + while True: + try: + count = method(*args, retry=retry) + total += count + except Timeout as timeout: + total += timeout.args[0] + else: + break + return total + + def stats(self, enable=True, reset=False): + """Return cache statistics hits and misses. + + :param bool enable: enable collecting statistics (default True) + :param bool reset: reset hits and misses to 0 (default False) + :return: (hits, misses) + + """ + results = [shard.stats(enable, reset) for shard in self._shards] + total_hits = sum(hits for hits, _ in results) + total_misses = sum(misses for _, misses in results) + return total_hits, total_misses + + def volume(self): + """Return estimated total size of cache on disk. + + :return: size in bytes + + """ + return sum(shard.volume() for shard in self._shards) + + def close(self): + """Close database connection.""" + for shard in self._shards: + shard.close() + self._caches.clear() + self._deques.clear() + self._indexes.clear() + + def __enter__(self): + return self + + def __exit__(self, *exception): + self.close() + + def __getstate__(self): + return (self._directory, self._count, self.timeout, type(self.disk)) + + def __setstate__(self, state): + self.__init__(*state) + + def __iter__(self): + """Iterate keys in cache including expired items.""" + iterators = (iter(shard) for shard in self._shards) + return it.chain.from_iterable(iterators) + + def __reversed__(self): + """Reverse iterate keys in cache including expired items.""" + iterators = (reversed(shard) for shard in reversed(self._shards)) + return it.chain.from_iterable(iterators) + + def __len__(self): + """Count of items in cache including expired items.""" + return sum(len(shard) for shard in self._shards) + + def reset(self, key, value=ENOVAL): + """Reset `key` and `value` item from Settings table. + + If `value` is not given, it is reloaded from the Settings + table. Otherwise, the Settings table is updated. + + Settings attributes on cache objects are lazy-loaded and + read-only. Use `reset` to update the value. + + Settings with the ``sqlite_`` prefix correspond to SQLite + pragmas. Updating the value will execute the corresponding PRAGMA + statement. + + :param str key: Settings key for item + :param value: value for item (optional) + :return: updated value for item + + """ + for shard in self._shards: + while True: + try: + result = shard.reset(key, value) + except Timeout: + pass + else: + break + return result + + def cache(self, name, timeout=60, disk=None, **settings): + """Return Cache with given `name` in subdirectory. + + If disk is none (default), uses the fanout cache disk. + + >>> fanout_cache = FanoutCache() + >>> cache = fanout_cache.cache('test') + >>> cache.set('abc', 123) + True + >>> cache.get('abc') + 123 + >>> len(cache) + 1 + >>> cache.delete('abc') + True + + :param str name: subdirectory name for Cache + :param float timeout: SQLite connection timeout + :param disk: Disk type or subclass for serialization + :param settings: any of DEFAULT_SETTINGS + :return: Cache with given name + + """ + _caches = self._caches + + try: + return _caches[name] + except KeyError: + parts = name.split('/') + directory = op.join(self._directory, 'cache', *parts) + temp = Cache( + directory=directory, + timeout=timeout, + disk=self._disk if disk is None else Disk, + **settings, + ) + _caches[name] = temp + return temp + + def deque(self, name, maxlen=None): + """Return Deque with given `name` in subdirectory. + + >>> cache = FanoutCache() + >>> deque = cache.deque('test') + >>> deque.extend('abc') + >>> deque.popleft() + 'a' + >>> deque.pop() + 'c' + >>> len(deque) + 1 + + :param str name: subdirectory name for Deque + :param maxlen: max length (default None, no max) + :return: Deque with given name + + """ + _deques = self._deques + + try: + return _deques[name] + except KeyError: + parts = name.split('/') + directory = op.join(self._directory, 'deque', *parts) + cache = Cache( + directory=directory, + disk=self._disk, + eviction_policy='none', + ) + deque = Deque.fromcache(cache, maxlen=maxlen) + _deques[name] = deque + return deque + + def index(self, name): + """Return Index with given `name` in subdirectory. + + >>> cache = FanoutCache() + >>> index = cache.index('test') + >>> index['abc'] = 123 + >>> index['def'] = 456 + >>> index['ghi'] = 789 + >>> index.popitem() + ('ghi', 789) + >>> del index['abc'] + >>> len(index) + 1 + >>> index['def'] + 456 + + :param str name: subdirectory name for Index + :return: Index with given name + + """ + _indexes = self._indexes + + try: + return _indexes[name] + except KeyError: + parts = name.split('/') + directory = op.join(self._directory, 'index', *parts) + cache = Cache( + directory=directory, + disk=self._disk, + eviction_policy='none', + ) + index = Index.fromcache(cache) + _indexes[name] = index + return index + + +FanoutCache.memoize = Cache.memoize # type: ignore diff --git a/lib/python3.1/site-packages/diskcache/persistent.py b/lib/python3.1/site-packages/diskcache/persistent.py new file mode 100644 index 0000000000000000000000000000000000000000..522bb74c5b1efc5b270df384f2c80f616d5eb2f0 --- /dev/null +++ b/lib/python3.1/site-packages/diskcache/persistent.py @@ -0,0 +1,1245 @@ +"""Persistent Data Types +""" + +import operator as op +from collections import OrderedDict +from collections.abc import ( + ItemsView, + KeysView, + MutableMapping, + Sequence, + ValuesView, +) +from contextlib import contextmanager +from shutil import rmtree + +from .core import ENOVAL, Cache + + +def _make_compare(seq_op, doc): + """Make compare method with Sequence semantics.""" + + def compare(self, that): + """Compare method for deque and sequence.""" + if not isinstance(that, Sequence): + return NotImplemented + + len_self = len(self) + len_that = len(that) + + if len_self != len_that: + if seq_op is op.eq: + return False + if seq_op is op.ne: + return True + + for alpha, beta in zip(self, that): + if alpha != beta: + return seq_op(alpha, beta) + + return seq_op(len_self, len_that) + + compare.__name__ = '__{0}__'.format(seq_op.__name__) + doc_str = 'Return True if and only if deque is {0} `that`.' + compare.__doc__ = doc_str.format(doc) + + return compare + + +class Deque(Sequence): + """Persistent sequence with double-ended queue semantics. + + Double-ended queue is an ordered collection with optimized access at its + endpoints. + + Items are serialized to disk. Deque may be initialized from directory path + where items are stored. + + >>> deque = Deque() + >>> deque += range(5) + >>> list(deque) + [0, 1, 2, 3, 4] + >>> for value in range(5): + ... deque.appendleft(-value) + >>> len(deque) + 10 + >>> list(deque) + [-4, -3, -2, -1, 0, 0, 1, 2, 3, 4] + >>> deque.pop() + 4 + >>> deque.popleft() + -4 + >>> deque.reverse() + >>> list(deque) + [3, 2, 1, 0, 0, -1, -2, -3] + + """ + + def __init__(self, iterable=(), directory=None, maxlen=None): + """Initialize deque instance. + + If directory is None then temporary directory created. The directory + will *not* be automatically removed. + + :param iterable: iterable of items to append to deque + :param directory: deque directory (default None) + + """ + self._cache = Cache(directory, eviction_policy='none') + self._maxlen = float('inf') if maxlen is None else maxlen + self._extend(iterable) + + @classmethod + def fromcache(cls, cache, iterable=(), maxlen=None): + """Initialize deque using `cache`. + + >>> cache = Cache() + >>> deque = Deque.fromcache(cache, [5, 6, 7, 8]) + >>> deque.cache is cache + True + >>> len(deque) + 4 + >>> 7 in deque + True + >>> deque.popleft() + 5 + + :param Cache cache: cache to use + :param iterable: iterable of items + :return: initialized Deque + + """ + # pylint: disable=no-member,protected-access + self = cls.__new__(cls) + self._cache = cache + self._maxlen = float('inf') if maxlen is None else maxlen + self._extend(iterable) + return self + + @property + def cache(self): + """Cache used by deque.""" + return self._cache + + @property + def directory(self): + """Directory path where deque is stored.""" + return self._cache.directory + + @property + def maxlen(self): + """Max length of the deque.""" + return self._maxlen + + @maxlen.setter + def maxlen(self, value): + """Set max length of the deque. + + Pops items from left while length greater than max. + + >>> deque = Deque() + >>> deque.extendleft('abcde') + >>> deque.maxlen = 3 + >>> list(deque) + ['c', 'd', 'e'] + + :param value: max length + + """ + self._maxlen = value + with self._cache.transact(retry=True): + while len(self._cache) > self._maxlen: + self._popleft() + + def _index(self, index, func): + len_self = len(self) + + if index >= 0: + if index >= len_self: + raise IndexError('deque index out of range') + + for key in self._cache.iterkeys(): + if index == 0: + try: + return func(key) + except KeyError: + continue + index -= 1 + else: + if index < -len_self: + raise IndexError('deque index out of range') + + index += 1 + + for key in self._cache.iterkeys(reverse=True): + if index == 0: + try: + return func(key) + except KeyError: + continue + index += 1 + + raise IndexError('deque index out of range') + + def __getitem__(self, index): + """deque.__getitem__(index) <==> deque[index] + + Return corresponding item for `index` in deque. + + See also `Deque.peekleft` and `Deque.peek` for indexing deque at index + ``0`` or ``-1``. + + >>> deque = Deque() + >>> deque.extend('abcde') + >>> deque[1] + 'b' + >>> deque[-2] + 'd' + + :param int index: index of item + :return: corresponding item + :raises IndexError: if index out of range + + """ + return self._index(index, self._cache.__getitem__) + + def __setitem__(self, index, value): + """deque.__setitem__(index, value) <==> deque[index] = value + + Store `value` in deque at `index`. + + >>> deque = Deque() + >>> deque.extend([None] * 3) + >>> deque[0] = 'a' + >>> deque[1] = 'b' + >>> deque[-1] = 'c' + >>> ''.join(deque) + 'abc' + + :param int index: index of value + :param value: value to store + :raises IndexError: if index out of range + + """ + + def _set_value(key): + return self._cache.__setitem__(key, value) + + self._index(index, _set_value) + + def __delitem__(self, index): + """deque.__delitem__(index) <==> del deque[index] + + Delete item in deque at `index`. + + >>> deque = Deque() + >>> deque.extend([None] * 3) + >>> del deque[0] + >>> del deque[1] + >>> del deque[-1] + >>> len(deque) + 0 + + :param int index: index of item + :raises IndexError: if index out of range + + """ + self._index(index, self._cache.__delitem__) + + def __repr__(self): + """deque.__repr__() <==> repr(deque) + + Return string with printable representation of deque. + + """ + name = type(self).__name__ + return '{0}(directory={1!r})'.format(name, self.directory) + + __eq__ = _make_compare(op.eq, 'equal to') + __ne__ = _make_compare(op.ne, 'not equal to') + __lt__ = _make_compare(op.lt, 'less than') + __gt__ = _make_compare(op.gt, 'greater than') + __le__ = _make_compare(op.le, 'less than or equal to') + __ge__ = _make_compare(op.ge, 'greater than or equal to') + + def __iadd__(self, iterable): + """deque.__iadd__(iterable) <==> deque += iterable + + Extend back side of deque with items from iterable. + + :param iterable: iterable of items to append to deque + :return: deque with added items + + """ + self._extend(iterable) + return self + + def __iter__(self): + """deque.__iter__() <==> iter(deque) + + Return iterator of deque from front to back. + + """ + _cache = self._cache + + for key in _cache.iterkeys(): + try: + yield _cache[key] + except KeyError: + pass + + def __len__(self): + """deque.__len__() <==> len(deque) + + Return length of deque. + + """ + return len(self._cache) + + def __reversed__(self): + """deque.__reversed__() <==> reversed(deque) + + Return iterator of deque from back to front. + + >>> deque = Deque() + >>> deque.extend('abcd') + >>> iterator = reversed(deque) + >>> next(iterator) + 'd' + >>> list(iterator) + ['c', 'b', 'a'] + + """ + _cache = self._cache + + for key in _cache.iterkeys(reverse=True): + try: + yield _cache[key] + except KeyError: + pass + + def __getstate__(self): + return self.directory, self.maxlen + + def __setstate__(self, state): + directory, maxlen = state + self.__init__(directory=directory, maxlen=maxlen) + + def append(self, value): + """Add `value` to back of deque. + + >>> deque = Deque() + >>> deque.append('a') + >>> deque.append('b') + >>> deque.append('c') + >>> list(deque) + ['a', 'b', 'c'] + + :param value: value to add to back of deque + + """ + with self._cache.transact(retry=True): + self._cache.push(value, retry=True) + if len(self._cache) > self._maxlen: + self._popleft() + + _append = append + + def appendleft(self, value): + """Add `value` to front of deque. + + >>> deque = Deque() + >>> deque.appendleft('a') + >>> deque.appendleft('b') + >>> deque.appendleft('c') + >>> list(deque) + ['c', 'b', 'a'] + + :param value: value to add to front of deque + + """ + with self._cache.transact(retry=True): + self._cache.push(value, side='front', retry=True) + if len(self._cache) > self._maxlen: + self._pop() + + _appendleft = appendleft + + def clear(self): + """Remove all elements from deque. + + >>> deque = Deque('abc') + >>> len(deque) + 3 + >>> deque.clear() + >>> list(deque) + [] + + """ + self._cache.clear(retry=True) + + _clear = clear + + def copy(self): + """Copy deque with same directory and max length.""" + TypeSelf = type(self) + return TypeSelf(directory=self.directory, maxlen=self.maxlen) + + def count(self, value): + """Return number of occurrences of `value` in deque. + + >>> deque = Deque() + >>> deque += [num for num in range(1, 5) for _ in range(num)] + >>> deque.count(0) + 0 + >>> deque.count(1) + 1 + >>> deque.count(4) + 4 + + :param value: value to count in deque + :return: count of items equal to value in deque + + """ + return sum(1 for item in self if value == item) + + def extend(self, iterable): + """Extend back side of deque with values from `iterable`. + + :param iterable: iterable of values + + """ + for value in iterable: + self._append(value) + + _extend = extend + + def extendleft(self, iterable): + """Extend front side of deque with value from `iterable`. + + >>> deque = Deque() + >>> deque.extendleft('abc') + >>> list(deque) + ['c', 'b', 'a'] + + :param iterable: iterable of values + + """ + for value in iterable: + self._appendleft(value) + + def peek(self): + """Peek at value at back of deque. + + Faster than indexing deque at -1. + + If deque is empty then raise IndexError. + + >>> deque = Deque() + >>> deque.peek() + Traceback (most recent call last): + ... + IndexError: peek from an empty deque + >>> deque += 'abc' + >>> deque.peek() + 'c' + + :return: value at back of deque + :raises IndexError: if deque is empty + + """ + default = None, ENOVAL + _, value = self._cache.peek(default=default, side='back', retry=True) + if value is ENOVAL: + raise IndexError('peek from an empty deque') + return value + + def peekleft(self): + """Peek at value at front of deque. + + Faster than indexing deque at 0. + + If deque is empty then raise IndexError. + + >>> deque = Deque() + >>> deque.peekleft() + Traceback (most recent call last): + ... + IndexError: peek from an empty deque + >>> deque += 'abc' + >>> deque.peekleft() + 'a' + + :return: value at front of deque + :raises IndexError: if deque is empty + + """ + default = None, ENOVAL + _, value = self._cache.peek(default=default, side='front', retry=True) + if value is ENOVAL: + raise IndexError('peek from an empty deque') + return value + + def pop(self): + """Remove and return value at back of deque. + + If deque is empty then raise IndexError. + + >>> deque = Deque() + >>> deque += 'ab' + >>> deque.pop() + 'b' + >>> deque.pop() + 'a' + >>> deque.pop() + Traceback (most recent call last): + ... + IndexError: pop from an empty deque + + :return: value at back of deque + :raises IndexError: if deque is empty + + """ + default = None, ENOVAL + _, value = self._cache.pull(default=default, side='back', retry=True) + if value is ENOVAL: + raise IndexError('pop from an empty deque') + return value + + _pop = pop + + def popleft(self): + """Remove and return value at front of deque. + + >>> deque = Deque() + >>> deque += 'ab' + >>> deque.popleft() + 'a' + >>> deque.popleft() + 'b' + >>> deque.popleft() + Traceback (most recent call last): + ... + IndexError: pop from an empty deque + + :return: value at front of deque + :raises IndexError: if deque is empty + + """ + default = None, ENOVAL + _, value = self._cache.pull(default=default, retry=True) + if value is ENOVAL: + raise IndexError('pop from an empty deque') + return value + + _popleft = popleft + + def remove(self, value): + """Remove first occurrence of `value` in deque. + + >>> deque = Deque() + >>> deque += 'aab' + >>> deque.remove('a') + >>> list(deque) + ['a', 'b'] + >>> deque.remove('b') + >>> list(deque) + ['a'] + >>> deque.remove('c') + Traceback (most recent call last): + ... + ValueError: deque.remove(value): value not in deque + + :param value: value to remove + :raises ValueError: if value not in deque + + """ + _cache = self._cache + + for key in _cache.iterkeys(): + try: + item = _cache[key] + except KeyError: + continue + else: + if value == item: + try: + del _cache[key] + except KeyError: + continue + return + + raise ValueError('deque.remove(value): value not in deque') + + def reverse(self): + """Reverse deque in place. + + >>> deque = Deque() + >>> deque += 'abc' + >>> deque.reverse() + >>> list(deque) + ['c', 'b', 'a'] + + """ + # pylint: disable=protected-access + # GrantJ 2019-03-22 Consider using an algorithm that swaps the values + # at two keys. Like self._cache.swap(key1, key2, retry=True) The swap + # method would exchange the values at two given keys. Then, using a + # forward iterator and a reverse iterator, the reverse method could + # avoid making copies of the values. + temp = Deque(iterable=reversed(self)) + self._clear() + self._extend(temp) + directory = temp.directory + temp._cache.close() + del temp + rmtree(directory) + + def rotate(self, steps=1): + """Rotate deque right by `steps`. + + If steps is negative then rotate left. + + >>> deque = Deque() + >>> deque += range(5) + >>> deque.rotate(2) + >>> list(deque) + [3, 4, 0, 1, 2] + >>> deque.rotate(-1) + >>> list(deque) + [4, 0, 1, 2, 3] + + :param int steps: number of steps to rotate (default 1) + + """ + if not isinstance(steps, int): + type_name = type(steps).__name__ + raise TypeError('integer argument expected, got %s' % type_name) + + len_self = len(self) + + if not len_self: + return + + if steps >= 0: + steps %= len_self + + for _ in range(steps): + try: + value = self._pop() + except IndexError: + return + else: + self._appendleft(value) + else: + steps *= -1 + steps %= len_self + + for _ in range(steps): + try: + value = self._popleft() + except IndexError: + return + else: + self._append(value) + + __hash__ = None # type: ignore + + @contextmanager + def transact(self): + """Context manager to perform a transaction by locking the deque. + + While the deque is locked, no other write operation is permitted. + Transactions should therefore be as short as possible. Read and write + operations performed in a transaction are atomic. Read operations may + occur concurrent to a transaction. + + Transactions may be nested and may not be shared between threads. + + >>> from diskcache import Deque + >>> deque = Deque() + >>> deque += range(5) + >>> with deque.transact(): # Atomically rotate elements. + ... value = deque.pop() + ... deque.appendleft(value) + >>> list(deque) + [4, 0, 1, 2, 3] + + :return: context manager for use in `with` statement + + """ + with self._cache.transact(retry=True): + yield + + +class Index(MutableMapping): + """Persistent mutable mapping with insertion order iteration. + + Items are serialized to disk. Index may be initialized from directory path + where items are stored. + + Hashing protocol is not used. Keys are looked up by their serialized + format. See ``diskcache.Disk`` for details. + + >>> index = Index() + >>> index.update([('a', 1), ('b', 2), ('c', 3)]) + >>> index['a'] + 1 + >>> list(index) + ['a', 'b', 'c'] + >>> len(index) + 3 + >>> del index['b'] + >>> index.popitem() + ('c', 3) + + """ + + def __init__(self, *args, **kwargs): + """Initialize index in directory and update items. + + Optional first argument may be string specifying directory where items + are stored. When None or not given, temporary directory is created. + + >>> index = Index({'a': 1, 'b': 2, 'c': 3}) + >>> len(index) + 3 + >>> directory = index.directory + >>> inventory = Index(directory, d=4) + >>> inventory['b'] + 2 + >>> len(inventory) + 4 + + """ + if args and isinstance(args[0], (bytes, str)): + directory = args[0] + args = args[1:] + else: + if args and args[0] is None: + args = args[1:] + directory = None + self._cache = Cache(directory, eviction_policy='none') + self._update(*args, **kwargs) + + _update = MutableMapping.update + + @classmethod + def fromcache(cls, cache, *args, **kwargs): + """Initialize index using `cache` and update items. + + >>> cache = Cache() + >>> index = Index.fromcache(cache, {'a': 1, 'b': 2, 'c': 3}) + >>> index.cache is cache + True + >>> len(index) + 3 + >>> 'b' in index + True + >>> index['c'] + 3 + + :param Cache cache: cache to use + :param args: mapping or sequence of items + :param kwargs: mapping of items + :return: initialized Index + + """ + # pylint: disable=no-member,protected-access + self = cls.__new__(cls) + self._cache = cache + self._update(*args, **kwargs) + return self + + @property + def cache(self): + """Cache used by index.""" + return self._cache + + @property + def directory(self): + """Directory path where items are stored.""" + return self._cache.directory + + def __getitem__(self, key): + """index.__getitem__(key) <==> index[key] + + Return corresponding value for `key` in index. + + >>> index = Index() + >>> index.update({'a': 1, 'b': 2}) + >>> index['a'] + 1 + >>> index['b'] + 2 + >>> index['c'] + Traceback (most recent call last): + ... + KeyError: 'c' + + :param key: key for item + :return: value for item in index with given key + :raises KeyError: if key is not found + + """ + return self._cache[key] + + def __setitem__(self, key, value): + """index.__setitem__(key, value) <==> index[key] = value + + Set `key` and `value` item in index. + + >>> index = Index() + >>> index['a'] = 1 + >>> index[0] = None + >>> len(index) + 2 + + :param key: key for item + :param value: value for item + + """ + self._cache[key] = value + + def __delitem__(self, key): + """index.__delitem__(key) <==> del index[key] + + Delete corresponding item for `key` from index. + + >>> index = Index() + >>> index.update({'a': 1, 'b': 2}) + >>> del index['a'] + >>> del index['b'] + >>> len(index) + 0 + >>> del index['c'] + Traceback (most recent call last): + ... + KeyError: 'c' + + :param key: key for item + :raises KeyError: if key is not found + + """ + del self._cache[key] + + def setdefault(self, key, default=None): + """Set and get value for `key` in index using `default`. + + If `key` is not in index then set corresponding value to `default`. If + `key` is in index then ignore `default` and return existing value. + + >>> index = Index() + >>> index.setdefault('a', 0) + 0 + >>> index.setdefault('a', 1) + 0 + + :param key: key for item + :param default: value if key is missing (default None) + :return: value for item in index with given key + + """ + _cache = self._cache + while True: + try: + return _cache[key] + except KeyError: + _cache.add(key, default, retry=True) + + def peekitem(self, last=True): + """Peek at key and value item pair in index based on iteration order. + + >>> index = Index() + >>> for num, letter in enumerate('xyz'): + ... index[letter] = num + >>> index.peekitem() + ('z', 2) + >>> index.peekitem(last=False) + ('x', 0) + + :param bool last: last item in iteration order (default True) + :return: key and value item pair + :raises KeyError: if cache is empty + + """ + return self._cache.peekitem(last, retry=True) + + def pop(self, key, default=ENOVAL): + """Remove corresponding item for `key` from index and return value. + + If `key` is missing then return `default`. If `default` is `ENOVAL` + then raise KeyError. + + >>> index = Index({'a': 1, 'b': 2}) + >>> index.pop('a') + 1 + >>> index.pop('b') + 2 + >>> index.pop('c', default=3) + 3 + >>> index.pop('d') + Traceback (most recent call last): + ... + KeyError: 'd' + + :param key: key for item + :param default: return value if key is missing (default ENOVAL) + :return: value for item if key is found else default + :raises KeyError: if key is not found and default is ENOVAL + + """ + _cache = self._cache + value = _cache.pop(key, default=default, retry=True) + if value is ENOVAL: + raise KeyError(key) + return value + + def popitem(self, last=True): + """Remove and return item pair. + + Item pairs are returned in last-in-first-out (LIFO) order if last is + True else first-in-first-out (FIFO) order. LIFO order imitates a stack + and FIFO order imitates a queue. + + >>> index = Index() + >>> index.update([('a', 1), ('b', 2), ('c', 3)]) + >>> index.popitem() + ('c', 3) + >>> index.popitem(last=False) + ('a', 1) + >>> index.popitem() + ('b', 2) + >>> index.popitem() + Traceback (most recent call last): + ... + KeyError: 'dictionary is empty' + + :param bool last: pop last item pair (default True) + :return: key and value item pair + :raises KeyError: if index is empty + + """ + # pylint: disable=arguments-differ,unbalanced-tuple-unpacking + _cache = self._cache + + with _cache.transact(retry=True): + key, value = _cache.peekitem(last=last) + del _cache[key] + + return key, value + + def push(self, value, prefix=None, side='back'): + """Push `value` onto `side` of queue in index identified by `prefix`. + + When prefix is None, integer keys are used. Otherwise, string keys are + used in the format "prefix-integer". Integer starts at 500 trillion. + + Defaults to pushing value on back of queue. Set side to 'front' to push + value on front of queue. Side must be one of 'back' or 'front'. + + See also `Index.pull`. + + >>> index = Index() + >>> print(index.push('apples')) + 500000000000000 + >>> print(index.push('beans')) + 500000000000001 + >>> print(index.push('cherries', side='front')) + 499999999999999 + >>> index[500000000000001] + 'beans' + >>> index.push('dates', prefix='fruit') + 'fruit-500000000000000' + + :param value: value for item + :param str prefix: key prefix (default None, key is integer) + :param str side: either 'back' or 'front' (default 'back') + :return: key for item in cache + + """ + return self._cache.push(value, prefix, side, retry=True) + + def pull(self, prefix=None, default=(None, None), side='front'): + """Pull key and value item pair from `side` of queue in index. + + When prefix is None, integer keys are used. Otherwise, string keys are + used in the format "prefix-integer". Integer starts at 500 trillion. + + If queue is empty, return default. + + Defaults to pulling key and value item pairs from front of queue. Set + side to 'back' to pull from back of queue. Side must be one of 'front' + or 'back'. + + See also `Index.push`. + + >>> index = Index() + >>> for letter in 'abc': + ... print(index.push(letter)) + 500000000000000 + 500000000000001 + 500000000000002 + >>> key, value = index.pull() + >>> print(key) + 500000000000000 + >>> value + 'a' + >>> _, value = index.pull(side='back') + >>> value + 'c' + >>> index.pull(prefix='fruit') + (None, None) + + :param str prefix: key prefix (default None, key is integer) + :param default: value to return if key is missing + (default (None, None)) + :param str side: either 'front' or 'back' (default 'front') + :return: key and value item pair or default if queue is empty + + """ + return self._cache.pull(prefix, default, side, retry=True) + + def clear(self): + """Remove all items from index. + + >>> index = Index({'a': 0, 'b': 1, 'c': 2}) + >>> len(index) + 3 + >>> index.clear() + >>> dict(index) + {} + + """ + self._cache.clear(retry=True) + + def __iter__(self): + """index.__iter__() <==> iter(index) + + Return iterator of index keys in insertion order. + + """ + return iter(self._cache) + + def __reversed__(self): + """index.__reversed__() <==> reversed(index) + + Return iterator of index keys in reversed insertion order. + + >>> index = Index() + >>> index.update([('a', 1), ('b', 2), ('c', 3)]) + >>> iterator = reversed(index) + >>> next(iterator) + 'c' + >>> list(iterator) + ['b', 'a'] + + """ + return reversed(self._cache) + + def __len__(self): + """index.__len__() <==> len(index) + + Return length of index. + + """ + return len(self._cache) + + def keys(self): + """Set-like object providing a view of index keys. + + >>> index = Index() + >>> index.update({'a': 1, 'b': 2, 'c': 3}) + >>> keys_view = index.keys() + >>> 'b' in keys_view + True + + :return: keys view + + """ + return KeysView(self) + + def values(self): + """Set-like object providing a view of index values. + + >>> index = Index() + >>> index.update({'a': 1, 'b': 2, 'c': 3}) + >>> values_view = index.values() + >>> 2 in values_view + True + + :return: values view + + """ + return ValuesView(self) + + def items(self): + """Set-like object providing a view of index items. + + >>> index = Index() + >>> index.update({'a': 1, 'b': 2, 'c': 3}) + >>> items_view = index.items() + >>> ('b', 2) in items_view + True + + :return: items view + + """ + return ItemsView(self) + + __hash__ = None # type: ignore + + def __getstate__(self): + return self.directory + + def __setstate__(self, state): + self.__init__(state) + + def __eq__(self, other): + """index.__eq__(other) <==> index == other + + Compare equality for index and `other`. + + Comparison to another index or ordered dictionary is + order-sensitive. Comparison to all other mappings is order-insensitive. + + >>> index = Index() + >>> pairs = [('a', 1), ('b', 2), ('c', 3)] + >>> index.update(pairs) + >>> from collections import OrderedDict + >>> od = OrderedDict(pairs) + >>> index == od + True + >>> index == {'c': 3, 'b': 2, 'a': 1} + True + + :param other: other mapping in equality comparison + :return: True if index equals other + + """ + if len(self) != len(other): + return False + + if isinstance(other, (Index, OrderedDict)): + alpha = ((key, self[key]) for key in self) + beta = ((key, other[key]) for key in other) + pairs = zip(alpha, beta) + return not any(a != x or b != y for (a, b), (x, y) in pairs) + else: + return all(self[key] == other.get(key, ENOVAL) for key in self) + + def __ne__(self, other): + """index.__ne__(other) <==> index != other + + Compare inequality for index and `other`. + + Comparison to another index or ordered dictionary is + order-sensitive. Comparison to all other mappings is order-insensitive. + + >>> index = Index() + >>> index.update([('a', 1), ('b', 2), ('c', 3)]) + >>> from collections import OrderedDict + >>> od = OrderedDict([('c', 3), ('b', 2), ('a', 1)]) + >>> index != od + True + >>> index != {'a': 1, 'b': 2} + True + + :param other: other mapping in inequality comparison + :return: True if index does not equal other + + """ + return not self == other + + def memoize(self, name=None, typed=False, ignore=()): + """Memoizing cache decorator. + + Decorator to wrap callable with memoizing function using cache. + Repeated calls with the same arguments will lookup result in cache and + avoid function evaluation. + + If name is set to None (default), the callable name will be determined + automatically. + + If typed is set to True, function arguments of different types will be + cached separately. For example, f(3) and f(3.0) will be treated as + distinct calls with distinct results. + + The original underlying function is accessible through the __wrapped__ + attribute. This is useful for introspection, for bypassing the cache, + or for rewrapping the function with a different cache. + + >>> from diskcache import Index + >>> mapping = Index() + >>> @mapping.memoize() + ... def fibonacci(number): + ... if number == 0: + ... return 0 + ... elif number == 1: + ... return 1 + ... else: + ... return fibonacci(number - 1) + fibonacci(number - 2) + >>> print(fibonacci(100)) + 354224848179261915075 + + An additional `__cache_key__` attribute can be used to generate the + cache key used for the given arguments. + + >>> key = fibonacci.__cache_key__(100) + >>> print(mapping[key]) + 354224848179261915075 + + Remember to call memoize when decorating a callable. If you forget, + then a TypeError will occur. Note the lack of parenthenses after + memoize below: + + >>> @mapping.memoize + ... def test(): + ... pass + Traceback (most recent call last): + ... + TypeError: name cannot be callable + + :param str name: name given for callable (default None, automatic) + :param bool typed: cache different types separately (default False) + :param set ignore: positional or keyword args to ignore (default ()) + :return: callable decorator + + """ + return self._cache.memoize(name, typed, ignore=ignore) + + @contextmanager + def transact(self): + """Context manager to perform a transaction by locking the index. + + While the index is locked, no other write operation is permitted. + Transactions should therefore be as short as possible. Read and write + operations performed in a transaction are atomic. Read operations may + occur concurrent to a transaction. + + Transactions may be nested and may not be shared between threads. + + >>> from diskcache import Index + >>> mapping = Index() + >>> with mapping.transact(): # Atomically increment two keys. + ... mapping['total'] = mapping.get('total', 0) + 123.4 + ... mapping['count'] = mapping.get('count', 0) + 1 + >>> with mapping.transact(): # Atomically calculate average. + ... average = mapping['total'] / mapping['count'] + >>> average + 123.4 + + :return: context manager for use in `with` statement + + """ + with self._cache.transact(retry=True): + yield + + def __repr__(self): + """index.__repr__() <==> repr(index) + + Return string with printable representation of index. + + """ + name = type(self).__name__ + return '{0}({1!r})'.format(name, self.directory) diff --git a/lib/python3.1/site-packages/diskcache/recipes.py b/lib/python3.1/site-packages/diskcache/recipes.py new file mode 100644 index 0000000000000000000000000000000000000000..babb68f67bd38f40b7fa31fbeaf4ab13ac762c65 --- /dev/null +++ b/lib/python3.1/site-packages/diskcache/recipes.py @@ -0,0 +1,488 @@ +"""Disk Cache Recipes +""" + +import functools +import math +import os +import random +import threading +import time + +from .core import ENOVAL, args_to_key, full_name + + +class Averager: + """Recipe for calculating a running average. + + Sometimes known as "online statistics," the running average maintains the + total and count. The average can then be calculated at any time. + + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + + >>> import diskcache + >>> cache = diskcache.FanoutCache() + >>> ave = Averager(cache, 'latency') + >>> ave.add(0.080) + >>> ave.add(0.120) + >>> ave.get() + 0.1 + >>> ave.add(0.160) + >>> ave.pop() + 0.12 + >>> print(ave.get()) + None + + """ + + def __init__(self, cache, key, expire=None, tag=None): + self._cache = cache + self._key = key + self._expire = expire + self._tag = tag + + def add(self, value): + """Add `value` to average.""" + with self._cache.transact(retry=True): + total, count = self._cache.get(self._key, default=(0.0, 0)) + total += value + count += 1 + self._cache.set( + self._key, + (total, count), + expire=self._expire, + tag=self._tag, + ) + + def get(self): + """Get current average or return `None` if count equals zero.""" + total, count = self._cache.get(self._key, default=(0.0, 0), retry=True) + return None if count == 0 else total / count + + def pop(self): + """Return current average and delete key.""" + total, count = self._cache.pop(self._key, default=(0.0, 0), retry=True) + return None if count == 0 else total / count + + +class Lock: + """Recipe for cross-process and cross-thread lock. + + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + + >>> import diskcache + >>> cache = diskcache.Cache() + >>> lock = Lock(cache, 'report-123') + >>> lock.acquire() + >>> lock.release() + >>> with lock: + ... pass + + """ + + def __init__(self, cache, key, expire=None, tag=None): + self._cache = cache + self._key = key + self._expire = expire + self._tag = tag + + def acquire(self): + """Acquire lock using spin-lock algorithm.""" + while True: + added = self._cache.add( + self._key, + None, + expire=self._expire, + tag=self._tag, + retry=True, + ) + if added: + break + time.sleep(0.001) + + def release(self): + """Release lock by deleting key.""" + self._cache.delete(self._key, retry=True) + + def locked(self): + """Return true if the lock is acquired.""" + return self._key in self._cache + + def __enter__(self): + self.acquire() + + def __exit__(self, *exc_info): + self.release() + + +class RLock: + """Recipe for cross-process and cross-thread re-entrant lock. + + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + + >>> import diskcache + >>> cache = diskcache.Cache() + >>> rlock = RLock(cache, 'user-123') + >>> rlock.acquire() + >>> rlock.acquire() + >>> rlock.release() + >>> with rlock: + ... pass + >>> rlock.release() + >>> rlock.release() + Traceback (most recent call last): + ... + AssertionError: cannot release un-acquired lock + + """ + + def __init__(self, cache, key, expire=None, tag=None): + self._cache = cache + self._key = key + self._expire = expire + self._tag = tag + + def acquire(self): + """Acquire lock by incrementing count using spin-lock algorithm.""" + pid = os.getpid() + tid = threading.get_ident() + pid_tid = '{}-{}'.format(pid, tid) + + while True: + with self._cache.transact(retry=True): + value, count = self._cache.get(self._key, default=(None, 0)) + if pid_tid == value or count == 0: + self._cache.set( + self._key, + (pid_tid, count + 1), + expire=self._expire, + tag=self._tag, + ) + return + time.sleep(0.001) + + def release(self): + """Release lock by decrementing count.""" + pid = os.getpid() + tid = threading.get_ident() + pid_tid = '{}-{}'.format(pid, tid) + + with self._cache.transact(retry=True): + value, count = self._cache.get(self._key, default=(None, 0)) + is_owned = pid_tid == value and count > 0 + assert is_owned, 'cannot release un-acquired lock' + self._cache.set( + self._key, + (value, count - 1), + expire=self._expire, + tag=self._tag, + ) + + def __enter__(self): + self.acquire() + + def __exit__(self, *exc_info): + self.release() + + +class BoundedSemaphore: + """Recipe for cross-process and cross-thread bounded semaphore. + + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + + >>> import diskcache + >>> cache = diskcache.Cache() + >>> semaphore = BoundedSemaphore(cache, 'max-cons', value=2) + >>> semaphore.acquire() + >>> semaphore.acquire() + >>> semaphore.release() + >>> with semaphore: + ... pass + >>> semaphore.release() + >>> semaphore.release() + Traceback (most recent call last): + ... + AssertionError: cannot release un-acquired semaphore + + """ + + def __init__(self, cache, key, value=1, expire=None, tag=None): + self._cache = cache + self._key = key + self._value = value + self._expire = expire + self._tag = tag + + def acquire(self): + """Acquire semaphore by decrementing value using spin-lock algorithm.""" + while True: + with self._cache.transact(retry=True): + value = self._cache.get(self._key, default=self._value) + if value > 0: + self._cache.set( + self._key, + value - 1, + expire=self._expire, + tag=self._tag, + ) + return + time.sleep(0.001) + + def release(self): + """Release semaphore by incrementing value.""" + with self._cache.transact(retry=True): + value = self._cache.get(self._key, default=self._value) + assert self._value > value, 'cannot release un-acquired semaphore' + value += 1 + self._cache.set( + self._key, + value, + expire=self._expire, + tag=self._tag, + ) + + def __enter__(self): + self.acquire() + + def __exit__(self, *exc_info): + self.release() + + +def throttle( + cache, + count, + seconds, + name=None, + expire=None, + tag=None, + time_func=time.time, + sleep_func=time.sleep, +): + """Decorator to throttle calls to function. + + Assumes keys will not be evicted. Set the eviction policy to 'none' on the + cache to guarantee the keys are not evicted. + + >>> import diskcache, time + >>> cache = diskcache.Cache() + >>> count = 0 + >>> @throttle(cache, 2, 1) # 2 calls per 1 second + ... def increment(): + ... global count + ... count += 1 + >>> start = time.time() + >>> while (time.time() - start) <= 2: + ... increment() + >>> count in (6, 7) # 6 or 7 calls depending on CPU load + True + + """ + + def decorator(func): + rate = count / float(seconds) + key = full_name(func) if name is None else name + now = time_func() + cache.set(key, (now, count), expire=expire, tag=tag, retry=True) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + while True: + with cache.transact(retry=True): + last, tally = cache.get(key) + now = time_func() + tally += (now - last) * rate + delay = 0 + + if tally > count: + cache.set(key, (now, count - 1), expire) + elif tally >= 1: + cache.set(key, (now, tally - 1), expire) + else: + delay = (1 - tally) / rate + + if delay: + sleep_func(delay) + else: + break + + return func(*args, **kwargs) + + return wrapper + + return decorator + + +def barrier(cache, lock_factory, name=None, expire=None, tag=None): + """Barrier to calling decorated function. + + Supports different kinds of locks: Lock, RLock, BoundedSemaphore. + + Assumes keys will not be evicted. Set the eviction policy to 'none' on the + cache to guarantee the keys are not evicted. + + >>> import diskcache, time + >>> cache = diskcache.Cache() + >>> @barrier(cache, Lock) + ... def work(num): + ... print('worker started') + ... time.sleep(1) + ... print('worker finished') + >>> import multiprocessing.pool + >>> pool = multiprocessing.pool.ThreadPool(2) + >>> _ = pool.map(work, range(2)) + worker started + worker finished + worker started + worker finished + >>> pool.terminate() + + """ + + def decorator(func): + key = full_name(func) if name is None else name + lock = lock_factory(cache, key, expire=expire, tag=tag) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + with lock: + return func(*args, **kwargs) + + return wrapper + + return decorator + + +def memoize_stampede( + cache, expire, name=None, typed=False, tag=None, beta=1, ignore=() +): + """Memoizing cache decorator with cache stampede protection. + + Cache stampedes are a type of system overload that can occur when parallel + computing systems using memoization come under heavy load. This behaviour + is sometimes also called dog-piling, cache miss storm, cache choking, or + the thundering herd problem. + + The memoization decorator implements cache stampede protection through + early recomputation. Early recomputation of function results will occur + probabilistically before expiration in a background thread of + execution. Early probabilistic recomputation is based on research by + Vattani, A.; Chierichetti, F.; Lowenstein, K. (2015), Optimal Probabilistic + Cache Stampede Prevention, VLDB, pp. 886-897, ISSN 2150-8097 + + If name is set to None (default), the callable name will be determined + automatically. + + If typed is set to True, function arguments of different types will be + cached separately. For example, f(3) and f(3.0) will be treated as distinct + calls with distinct results. + + The original underlying function is accessible through the `__wrapped__` + attribute. This is useful for introspection, for bypassing the cache, or + for rewrapping the function with a different cache. + + >>> from diskcache import Cache + >>> cache = Cache() + >>> @memoize_stampede(cache, expire=1) + ... def fib(number): + ... if number == 0: + ... return 0 + ... elif number == 1: + ... return 1 + ... else: + ... return fib(number - 1) + fib(number - 2) + >>> print(fib(100)) + 354224848179261915075 + + An additional `__cache_key__` attribute can be used to generate the cache + key used for the given arguments. + + >>> key = fib.__cache_key__(100) + >>> del cache[key] + + Remember to call memoize when decorating a callable. If you forget, then a + TypeError will occur. + + :param cache: cache to store callable arguments and return values + :param float expire: seconds until arguments expire + :param str name: name given for callable (default None, automatic) + :param bool typed: cache different types separately (default False) + :param str tag: text to associate with arguments (default None) + :param set ignore: positional or keyword args to ignore (default ()) + :return: callable decorator + + """ + # Caution: Nearly identical code exists in Cache.memoize + def decorator(func): + """Decorator created by memoize call for callable.""" + base = (full_name(func),) if name is None else (name,) + + def timer(*args, **kwargs): + """Time execution of `func` and return result and time delta.""" + start = time.time() + result = func(*args, **kwargs) + delta = time.time() - start + return result, delta + + @functools.wraps(func) + def wrapper(*args, **kwargs): + """Wrapper for callable to cache arguments and return values.""" + key = wrapper.__cache_key__(*args, **kwargs) + pair, expire_time = cache.get( + key, + default=ENOVAL, + expire_time=True, + retry=True, + ) + + if pair is not ENOVAL: + result, delta = pair + now = time.time() + ttl = expire_time - now + + if (-delta * beta * math.log(random.random())) < ttl: + return result # Cache hit. + + # Check whether a thread has started for early recomputation. + + thread_key = key + (ENOVAL,) + thread_added = cache.add( + thread_key, + None, + expire=delta, + retry=True, + ) + + if thread_added: + # Start thread for early recomputation. + def recompute(): + with cache: + pair = timer(*args, **kwargs) + cache.set( + key, + pair, + expire=expire, + tag=tag, + retry=True, + ) + + thread = threading.Thread(target=recompute) + thread.daemon = True + thread.start() + + return result + + pair = timer(*args, **kwargs) + cache.set(key, pair, expire=expire, tag=tag, retry=True) + return pair[0] + + def __cache_key__(*args, **kwargs): + """Make key for cache given function arguments.""" + return args_to_key(base, args, kwargs, typed, ignore) + + wrapper.__cache_key__ = __cache_key__ + return wrapper + + return decorator diff --git a/lib/python3.1/site-packages/example.py b/lib/python3.1/site-packages/example.py new file mode 100644 index 0000000000000000000000000000000000000000..60173d51111e4b677b84251afc642d47bc63c193 --- /dev/null +++ b/lib/python3.1/site-packages/example.py @@ -0,0 +1,169 @@ +################################################################################# +# Copyright (c) 2020, NVIDIA Corporation. All rights reserved. # +# # +# Redistribution and use in source and binary forms, with or without # +# modification, are permitted provided that the following conditions are met: # +# # +# * Redistributions of source code must retain the above copyright notice, # +# this list of conditions and the following disclaimer. # +# * Redistributions in binary form must reproduce the above copyright # +# notice, this list of conditions and the following disclaimer in the # +# documentation and/or other materials provided with the distribution. # +# * Neither the name of the NVIDIA Corporation nor the names of its # +# contributors may be used to endorse or promote products derived from # +# this software without specific prior written permission. # +# # +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE # +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # +# THE POSSIBILITY OF SUCH DAMAGE. # +################################################################################# + +# +# Sample script to demonstrate the usage of NVML API python bindings +# + +# To Run: +# $ python ./example.py + +from pynvml import * + +# +# Helper function +# +def StrVirt(mode): + if mode == NVML_GPU_VIRTUALIZATION_MODE_NONE: + return "None"; + elif mode == NVML_GPU_VIRTUALIZATION_MODE_PASSTHROUGH: + return "Pass-Through"; + elif mode == NVML_GPU_VIRTUALIZATION_MODE_VGPU: + return "VGPU"; + elif mode == NVML_GPU_VIRTUALIZATION_MODE_HOST_VGPU: + return "Host VGPU"; + elif mode == NVML_GPU_VIRTUALIZATION_MODE_HOST_VSGA: + return "Host VSGA"; + else: + return "Unknown"; + +# +# Converts errors into string messages +# +def handleError(err): + if (err.value == NVML_ERROR_NOT_SUPPORTED): + return "N/A" + else: + return err.__str__() + +####### +def deviceQuery(): + + strResult = '' + try: + # + # Initialize NVML + # + nvmlInit() + + strResult += ' ' + str(nvmlSystemGetDriverVersion()) + '\n' + + deviceCount = nvmlDeviceGetCount() + strResult += ' ' + str(deviceCount) + '\n' + + for i in range(0, deviceCount): + handle = nvmlDeviceGetHandleByIndex(i) + + pciInfo = nvmlDeviceGetPciInfo(handle) + + strResult += ' \n' % pciInfo.busId + + strResult += ' ' + nvmlDeviceGetName(handle) + '\n' + + brandNames = {NVML_BRAND_UNKNOWN : "Unknown", + NVML_BRAND_QUADRO : "Quadro", + NVML_BRAND_TESLA : "Tesla", + NVML_BRAND_NVS : "NVS", + NVML_BRAND_GRID : "Grid", + NVML_BRAND_TITAN : "Titan", + NVML_BRAND_GEFORCE : "GeForce", + NVML_BRAND_NVIDIA_VAPPS : "NVIDIA Virtual Applications", + NVML_BRAND_NVIDIA_VPC : "NVIDIA Virtual PC", + NVML_BRAND_NVIDIA_VCS : "NVIDIA Virtual Compute Server", + NVML_BRAND_NVIDIA_VWS : "NVIDIA RTX Virtual Workstation", + NVML_BRAND_NVIDIA_CLOUD_GAMING : "NVIDIA Cloud Gaming", + NVML_BRAND_QUADRO_RTX : "Quadro RTX", + NVML_BRAND_NVIDIA_RTX : "NVIDIA RTX", + NVML_BRAND_NVIDIA : "NVIDIA", + NVML_BRAND_GEFORCE_RTX : "GeForce RTX", + NVML_BRAND_TITAN_RTX : "TITAN RTX", + + } + + try: + # If nvmlDeviceGetBrand() succeeds it is guaranteed to be in the dictionary + brandName = brandNames[nvmlDeviceGetBrand(handle)] + except NVMLError as err: + brandName = handleError(err) + + strResult += ' ' + brandName + '\n' + + try: + serial = nvmlDeviceGetSerial(handle) + except NVMLError as err: + serial = handleError(err) + + strResult += ' ' + serial + '\n' + + try: + uuid = nvmlDeviceGetUUID(handle) + except NVMLError as err: + uuid = handleError(err) + + strResult += ' ' + uuid + '\n' + + strResult += ' \n' + try: + mode = StrVirt(nvmlDeviceGetVirtualizationMode(handle)) + except NVMLError as err: + mode = handleError(err) + strResult += ' ' + mode + '\n' + strResult += ' \n' + + try: + gridLicensableFeatures = nvmlDeviceGetGridLicensableFeatures(handle) + if gridLicensableFeatures.isGridLicenseSupported == 1: + strResult += ' \n' + for i in range(gridLicensableFeatures.licensableFeaturesCount): + if gridLicensableFeatures.gridLicensableFeatures[i].featureState == 0: + if nvmlDeviceGetVirtualizationMode(handle) == NVML_GPU_VIRTUALIZATION_MODE_PASSTHROUGH: + strResult += ' ' + 'NVIDIA Virtual Applications' + '\n' + strResult += ' ' + 'Licensed' + '\n' + else: + strResult += ' ' + gridLicensableFeatures.gridLicensableFeatures[i].productName + '\n' + strResult += ' ' + 'Unlicensed' + '\n' + else: + strResult += ' ' + gridLicensableFeatures.gridLicensableFeatures[i].productName + '\n' + strResult += ' ' + 'Licensed' + '\n' + strResult += ' \n' + except NVMLError as err: + gridLicensableFeatures = handleError(err) + + strResult += ' \n' + + except NVMLError as err: + strResult += 'example.py: ' + err.__str__() + '\n' + + nvmlShutdown() + + return strResult + +# If this is not exectued when module is imported +if __name__ == "__main__": + print(deviceQuery()) + diff --git a/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/INSTALLER b/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/METADATA b/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..fe82f2cb304c21ba5d766fd4ebd4cda51cc332fe --- /dev/null +++ b/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/METADATA @@ -0,0 +1,575 @@ +Metadata-Version: 2.1 +Name: fastapi +Version: 0.116.1 +Summary: FastAPI framework, high performance, easy to learn, fast to code, ready for production +Author-Email: =?utf-8?q?Sebasti=C3=A1n_Ram=C3=ADrez?= +Classifier: Intended Audience :: Information Technology +Classifier: Intended Audience :: System Administrators +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python +Classifier: Topic :: Internet +Classifier: Topic :: Software Development :: Libraries :: Application Frameworks +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: Software Development +Classifier: Typing :: Typed +Classifier: Development Status :: 4 - Beta +Classifier: Environment :: Web Environment +Classifier: Framework :: AsyncIO +Classifier: Framework :: FastAPI +Classifier: Framework :: Pydantic +Classifier: Framework :: Pydantic :: 1 +Classifier: Framework :: Pydantic :: 2 +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers +Classifier: Topic :: Internet :: WWW/HTTP +Project-URL: Homepage, https://github.com/fastapi/fastapi +Project-URL: Documentation, https://fastapi.tiangolo.com/ +Project-URL: Repository, https://github.com/fastapi/fastapi +Project-URL: Issues, https://github.com/fastapi/fastapi/issues +Project-URL: Changelog, https://fastapi.tiangolo.com/release-notes/ +Requires-Python: >=3.8 +Requires-Dist: starlette<0.48.0,>=0.40.0 +Requires-Dist: pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4 +Requires-Dist: typing-extensions>=4.8.0 +Provides-Extra: standard +Requires-Dist: fastapi-cli[standard]>=0.0.8; extra == "standard" +Requires-Dist: httpx>=0.23.0; extra == "standard" +Requires-Dist: jinja2>=3.1.5; extra == "standard" +Requires-Dist: python-multipart>=0.0.18; extra == "standard" +Requires-Dist: email-validator>=2.0.0; extra == "standard" +Requires-Dist: uvicorn[standard]>=0.12.0; extra == "standard" +Provides-Extra: standard-no-fastapi-cloud-cli +Requires-Dist: fastapi-cli[standard-no-fastapi-cloud-cli]>=0.0.8; extra == "standard-no-fastapi-cloud-cli" +Requires-Dist: httpx>=0.23.0; extra == "standard-no-fastapi-cloud-cli" +Requires-Dist: jinja2>=3.1.5; extra == "standard-no-fastapi-cloud-cli" +Requires-Dist: python-multipart>=0.0.18; extra == "standard-no-fastapi-cloud-cli" +Requires-Dist: email-validator>=2.0.0; extra == "standard-no-fastapi-cloud-cli" +Requires-Dist: uvicorn[standard]>=0.12.0; extra == "standard-no-fastapi-cloud-cli" +Provides-Extra: all +Requires-Dist: fastapi-cli[standard]>=0.0.8; extra == "all" +Requires-Dist: httpx>=0.23.0; extra == "all" +Requires-Dist: jinja2>=3.1.5; extra == "all" +Requires-Dist: python-multipart>=0.0.18; extra == "all" +Requires-Dist: itsdangerous>=1.1.0; extra == "all" +Requires-Dist: pyyaml>=5.3.1; extra == "all" +Requires-Dist: ujson!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,>=4.0.1; extra == "all" +Requires-Dist: orjson>=3.2.1; extra == "all" +Requires-Dist: email-validator>=2.0.0; extra == "all" +Requires-Dist: uvicorn[standard]>=0.12.0; extra == "all" +Requires-Dist: pydantic-settings>=2.0.0; extra == "all" +Requires-Dist: pydantic-extra-types>=2.0.0; extra == "all" +Description-Content-Type: text/markdown + +

    + FastAPI +

    +

    + FastAPI framework, high performance, easy to learn, fast to code, ready for production +

    +

    + + Test + + + Coverage + + + Package version + + + Supported Python versions + +

    + +--- + +**Documentation**: https://fastapi.tiangolo.com + +**Source Code**: https://github.com/fastapi/fastapi + +--- + +FastAPI is a modern, fast (high-performance), web framework for building APIs with Python based on standard Python type hints. + +The key features are: + +* **Fast**: Very high performance, on par with **NodeJS** and **Go** (thanks to Starlette and Pydantic). [One of the fastest Python frameworks available](#performance). +* **Fast to code**: Increase the speed to develop features by about 200% to 300%. * +* **Fewer bugs**: Reduce about 40% of human (developer) induced errors. * +* **Intuitive**: Great editor support. Completion everywhere. Less time debugging. +* **Easy**: Designed to be easy to use and learn. Less time reading docs. +* **Short**: Minimize code duplication. Multiple features from each parameter declaration. Fewer bugs. +* **Robust**: Get production-ready code. With automatic interactive documentation. +* **Standards-based**: Based on (and fully compatible with) the open standards for APIs: OpenAPI (previously known as Swagger) and JSON Schema. + +* estimation based on tests on an internal development team, building production applications. + +## Sponsors + + + + + + + + + + + + + + + + + + + + + + +Other sponsors + +## Opinions + +"_[...] I'm using **FastAPI** a ton these days. [...] I'm actually planning to use it for all of my team's **ML services at Microsoft**. Some of them are getting integrated into the core **Windows** product and some **Office** products._" + +
    Kabir Khan - Microsoft (ref)
    + +--- + +"_We adopted the **FastAPI** library to spawn a **REST** server that can be queried to obtain **predictions**. [for Ludwig]_" + +
    Piero Molino, Yaroslav Dudin, and Sai Sumanth Miryala - Uber (ref)
    + +--- + +"_**Netflix** is pleased to announce the open-source release of our **crisis management** orchestration framework: **Dispatch**! [built with **FastAPI**]_" + +
    Kevin Glisson, Marc Vilanova, Forest Monsen - Netflix (ref)
    + +--- + +"_I’m over the moon excited about **FastAPI**. It’s so fun!_" + +
    Brian Okken - Python Bytes podcast host (ref)
    + +--- + +"_Honestly, what you've built looks super solid and polished. In many ways, it's what I wanted **Hug** to be - it's really inspiring to see someone build that._" + +
    Timothy Crosley - Hug creator (ref)
    + +--- + +"_If you're looking to learn one **modern framework** for building REST APIs, check out **FastAPI** [...] It's fast, easy to use and easy to learn [...]_" + +"_We've switched over to **FastAPI** for our **APIs** [...] I think you'll like it [...]_" + +
    Ines Montani - Matthew Honnibal - Explosion AI founders - spaCy creators (ref) - (ref)
    + +--- + +"_If anyone is looking to build a production Python API, I would highly recommend **FastAPI**. It is **beautifully designed**, **simple to use** and **highly scalable**, it has become a **key component** in our API first development strategy and is driving many automations and services such as our Virtual TAC Engineer._" + +
    Deon Pillsbury - Cisco (ref)
    + +--- + +## **Typer**, the FastAPI of CLIs + + + +If you are building a CLI app to be used in the terminal instead of a web API, check out **Typer**. + +**Typer** is FastAPI's little sibling. And it's intended to be the **FastAPI of CLIs**. ⌨️ 🚀 + +## Requirements + +FastAPI stands on the shoulders of giants: + +* Starlette for the web parts. +* Pydantic for the data parts. + +## Installation + +Create and activate a virtual environment and then install FastAPI: + +
    + +```console +$ pip install "fastapi[standard]" + +---> 100% +``` + +
    + +**Note**: Make sure you put `"fastapi[standard]"` in quotes to ensure it works in all terminals. + +## Example + +### Create it + +Create a file `main.py` with: + +```Python +from typing import Union + +from fastapi import FastAPI + +app = FastAPI() + + +@app.get("/") +def read_root(): + return {"Hello": "World"} + + +@app.get("/items/{item_id}") +def read_item(item_id: int, q: Union[str, None] = None): + return {"item_id": item_id, "q": q} +``` + +
    +Or use async def... + +If your code uses `async` / `await`, use `async def`: + +```Python hl_lines="9 14" +from typing import Union + +from fastapi import FastAPI + +app = FastAPI() + + +@app.get("/") +async def read_root(): + return {"Hello": "World"} + + +@app.get("/items/{item_id}") +async def read_item(item_id: int, q: Union[str, None] = None): + return {"item_id": item_id, "q": q} +``` + +**Note**: + +If you don't know, check the _"In a hurry?"_ section about `async` and `await` in the docs. + +
    + +### Run it + +Run the server with: + +
    + +```console +$ fastapi dev main.py + + ╭────────── FastAPI CLI - Development mode ───────────╮ + │ │ + │ Serving at: http://127.0.0.1:8000 │ + │ │ + │ API docs: http://127.0.0.1:8000/docs │ + │ │ + │ Running in development mode, for production use: │ + │ │ + │ fastapi run │ + │ │ + ╰─────────────────────────────────────────────────────╯ + +INFO: Will watch for changes in these directories: ['/home/user/code/awesomeapp'] +INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit) +INFO: Started reloader process [2248755] using WatchFiles +INFO: Started server process [2248757] +INFO: Waiting for application startup. +INFO: Application startup complete. +``` + +
    + +
    +About the command fastapi dev main.py... + +The command `fastapi dev` reads your `main.py` file, detects the **FastAPI** app in it, and starts a server using Uvicorn. + +By default, `fastapi dev` will start with auto-reload enabled for local development. + +You can read more about it in the FastAPI CLI docs. + +
    + +### Check it + +Open your browser at http://127.0.0.1:8000/items/5?q=somequery. + +You will see the JSON response as: + +```JSON +{"item_id": 5, "q": "somequery"} +``` + +You already created an API that: + +* Receives HTTP requests in the _paths_ `/` and `/items/{item_id}`. +* Both _paths_ take `GET` operations (also known as HTTP _methods_). +* The _path_ `/items/{item_id}` has a _path parameter_ `item_id` that should be an `int`. +* The _path_ `/items/{item_id}` has an optional `str` _query parameter_ `q`. + +### Interactive API docs + +Now go to http://127.0.0.1:8000/docs. + +You will see the automatic interactive API documentation (provided by Swagger UI): + +![Swagger UI](https://fastapi.tiangolo.com/img/index/index-01-swagger-ui-simple.png) + +### Alternative API docs + +And now, go to http://127.0.0.1:8000/redoc. + +You will see the alternative automatic documentation (provided by ReDoc): + +![ReDoc](https://fastapi.tiangolo.com/img/index/index-02-redoc-simple.png) + +## Example upgrade + +Now modify the file `main.py` to receive a body from a `PUT` request. + +Declare the body using standard Python types, thanks to Pydantic. + +```Python hl_lines="4 9-12 25-27" +from typing import Union + +from fastapi import FastAPI +from pydantic import BaseModel + +app = FastAPI() + + +class Item(BaseModel): + name: str + price: float + is_offer: Union[bool, None] = None + + +@app.get("/") +def read_root(): + return {"Hello": "World"} + + +@app.get("/items/{item_id}") +def read_item(item_id: int, q: Union[str, None] = None): + return {"item_id": item_id, "q": q} + + +@app.put("/items/{item_id}") +def update_item(item_id: int, item: Item): + return {"item_name": item.name, "item_id": item_id} +``` + +The `fastapi dev` server should reload automatically. + +### Interactive API docs upgrade + +Now go to http://127.0.0.1:8000/docs. + +* The interactive API documentation will be automatically updated, including the new body: + +![Swagger UI](https://fastapi.tiangolo.com/img/index/index-03-swagger-02.png) + +* Click on the button "Try it out", it allows you to fill the parameters and directly interact with the API: + +![Swagger UI interaction](https://fastapi.tiangolo.com/img/index/index-04-swagger-03.png) + +* Then click on the "Execute" button, the user interface will communicate with your API, send the parameters, get the results and show them on the screen: + +![Swagger UI interaction](https://fastapi.tiangolo.com/img/index/index-05-swagger-04.png) + +### Alternative API docs upgrade + +And now, go to http://127.0.0.1:8000/redoc. + +* The alternative documentation will also reflect the new query parameter and body: + +![ReDoc](https://fastapi.tiangolo.com/img/index/index-06-redoc-02.png) + +### Recap + +In summary, you declare **once** the types of parameters, body, etc. as function parameters. + +You do that with standard modern Python types. + +You don't have to learn a new syntax, the methods or classes of a specific library, etc. + +Just standard **Python**. + +For example, for an `int`: + +```Python +item_id: int +``` + +or for a more complex `Item` model: + +```Python +item: Item +``` + +...and with that single declaration you get: + +* Editor support, including: + * Completion. + * Type checks. +* Validation of data: + * Automatic and clear errors when the data is invalid. + * Validation even for deeply nested JSON objects. +* Conversion of input data: coming from the network to Python data and types. Reading from: + * JSON. + * Path parameters. + * Query parameters. + * Cookies. + * Headers. + * Forms. + * Files. +* Conversion of output data: converting from Python data and types to network data (as JSON): + * Convert Python types (`str`, `int`, `float`, `bool`, `list`, etc). + * `datetime` objects. + * `UUID` objects. + * Database models. + * ...and many more. +* Automatic interactive API documentation, including 2 alternative user interfaces: + * Swagger UI. + * ReDoc. + +--- + +Coming back to the previous code example, **FastAPI** will: + +* Validate that there is an `item_id` in the path for `GET` and `PUT` requests. +* Validate that the `item_id` is of type `int` for `GET` and `PUT` requests. + * If it is not, the client will see a useful, clear error. +* Check if there is an optional query parameter named `q` (as in `http://127.0.0.1:8000/items/foo?q=somequery`) for `GET` requests. + * As the `q` parameter is declared with `= None`, it is optional. + * Without the `None` it would be required (as is the body in the case with `PUT`). +* For `PUT` requests to `/items/{item_id}`, read the body as JSON: + * Check that it has a required attribute `name` that should be a `str`. + * Check that it has a required attribute `price` that has to be a `float`. + * Check that it has an optional attribute `is_offer`, that should be a `bool`, if present. + * All this would also work for deeply nested JSON objects. +* Convert from and to JSON automatically. +* Document everything with OpenAPI, that can be used by: + * Interactive documentation systems. + * Automatic client code generation systems, for many languages. +* Provide 2 interactive documentation web interfaces directly. + +--- + +We just scratched the surface, but you already get the idea of how it all works. + +Try changing the line with: + +```Python + return {"item_name": item.name, "item_id": item_id} +``` + +...from: + +```Python + ... "item_name": item.name ... +``` + +...to: + +```Python + ... "item_price": item.price ... +``` + +...and see how your editor will auto-complete the attributes and know their types: + +![editor support](https://fastapi.tiangolo.com/img/vscode-completion.png) + +For a more complete example including more features, see the Tutorial - User Guide. + +**Spoiler alert**: the tutorial - user guide includes: + +* Declaration of **parameters** from other different places as: **headers**, **cookies**, **form fields** and **files**. +* How to set **validation constraints** as `maximum_length` or `regex`. +* A very powerful and easy to use **Dependency Injection** system. +* Security and authentication, including support for **OAuth2** with **JWT tokens** and **HTTP Basic** auth. +* More advanced (but equally easy) techniques for declaring **deeply nested JSON models** (thanks to Pydantic). +* **GraphQL** integration with Strawberry and other libraries. +* Many extra features (thanks to Starlette) as: + * **WebSockets** + * extremely easy tests based on HTTPX and `pytest` + * **CORS** + * **Cookie Sessions** + * ...and more. + +## Performance + +Independent TechEmpower benchmarks show **FastAPI** applications running under Uvicorn as one of the fastest Python frameworks available, only below Starlette and Uvicorn themselves (used internally by FastAPI). (*) + +To understand more about it, see the section Benchmarks. + +## Dependencies + +FastAPI depends on Pydantic and Starlette. + +### `standard` Dependencies + +When you install FastAPI with `pip install "fastapi[standard]"` it comes with the `standard` group of optional dependencies: + +Used by Pydantic: + +* email-validator - for email validation. + +Used by Starlette: + +* httpx - Required if you want to use the `TestClient`. +* jinja2 - Required if you want to use the default template configuration. +* python-multipart - Required if you want to support form "parsing", with `request.form()`. + +Used by FastAPI: + +* uvicorn - for the server that loads and serves your application. This includes `uvicorn[standard]`, which includes some dependencies (e.g. `uvloop`) needed for high performance serving. +* `fastapi-cli[standard]` - to provide the `fastapi` command. + * This includes `fastapi-cloud-cli`, which allows you to deploy your FastAPI application to FastAPI Cloud. + +### Without `standard` Dependencies + +If you don't want to include the `standard` optional dependencies, you can install with `pip install fastapi` instead of `pip install "fastapi[standard]"`. + +### Without `fastapi-cloud-cli` + +If you want to install FastAPI with the standard dependencies but without the `fastapi-cloud-cli`, you can install with `pip install "fastapi[standard-no-fastapi-cloud-cli]"`. + +### Additional Optional Dependencies + +There are some additional dependencies you might want to install. + +Additional optional Pydantic dependencies: + +* pydantic-settings - for settings management. +* pydantic-extra-types - for extra types to be used with Pydantic. + +Additional optional FastAPI dependencies: + +* orjson - Required if you want to use `ORJSONResponse`. +* ujson - Required if you want to use `UJSONResponse`. + +## License + +This project is licensed under the terms of the MIT license. diff --git a/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/RECORD b/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..1c9899ab38c97e92f31c63ba43e7e081305aef48 --- /dev/null +++ b/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/RECORD @@ -0,0 +1,96 @@ +../../../bin/fastapi,sha256=8MdBQDewEcOZo9qhLh2X-vqsB5TdrEUyiFHLEDVdv5I,244 +fastapi-0.116.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +fastapi-0.116.1.dist-info/METADATA,sha256=pnu4s6rAsNuB66sYhB59UFxRuZv2ua6fbH_jUM6HM2k,28115 +fastapi-0.116.1.dist-info/RECORD,, +fastapi-0.116.1.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90 +fastapi-0.116.1.dist-info/entry_points.txt,sha256=GCf-WbIZxyGT4MUmrPGj1cOHYZoGsNPHAvNkT6hnGeA,61 +fastapi-0.116.1.dist-info/licenses/LICENSE,sha256=Tsif_IFIW5f-xYSy1KlhAy7v_oNEU4lP2cEnSQbMdE4,1086 +fastapi/__init__.py,sha256=-U8vW9K3Hy78v_3O0ECrEfMmPtSuHaA1yAql96bd8ts,1081 +fastapi/__main__.py,sha256=bKePXLdO4SsVSM6r9SVoLickJDcR2c0cTOxZRKq26YQ,37 +fastapi/__pycache__/__init__.cpython-310.pyc,, +fastapi/__pycache__/__main__.cpython-310.pyc,, +fastapi/__pycache__/_compat.cpython-310.pyc,, +fastapi/__pycache__/applications.cpython-310.pyc,, +fastapi/__pycache__/background.cpython-310.pyc,, +fastapi/__pycache__/cli.cpython-310.pyc,, +fastapi/__pycache__/concurrency.cpython-310.pyc,, +fastapi/__pycache__/datastructures.cpython-310.pyc,, +fastapi/__pycache__/encoders.cpython-310.pyc,, +fastapi/__pycache__/exception_handlers.cpython-310.pyc,, +fastapi/__pycache__/exceptions.cpython-310.pyc,, +fastapi/__pycache__/logger.cpython-310.pyc,, +fastapi/__pycache__/param_functions.cpython-310.pyc,, +fastapi/__pycache__/params.cpython-310.pyc,, +fastapi/__pycache__/requests.cpython-310.pyc,, +fastapi/__pycache__/responses.cpython-310.pyc,, +fastapi/__pycache__/routing.cpython-310.pyc,, +fastapi/__pycache__/staticfiles.cpython-310.pyc,, +fastapi/__pycache__/templating.cpython-310.pyc,, +fastapi/__pycache__/testclient.cpython-310.pyc,, +fastapi/__pycache__/types.cpython-310.pyc,, +fastapi/__pycache__/utils.cpython-310.pyc,, +fastapi/__pycache__/websockets.cpython-310.pyc,, +fastapi/_compat.py,sha256=PwGTZd6d-u2o6YF9M8pQahuBtD_3q3Kpj7vU5-ngChc,24228 +fastapi/applications.py,sha256=rZTr0Ix-vdMwh6MQGCI_NC-Ir9lpfIGHHBY-JnNWZ_E,176550 +fastapi/background.py,sha256=rouLirxUANrcYC824MSMypXL_Qb2HYg2YZqaiEqbEKI,1768 +fastapi/cli.py,sha256=OYhZb0NR_deuT5ofyPF2NoNBzZDNOP8Salef2nk-HqA,418 +fastapi/concurrency.py,sha256=MirfowoSpkMQZ8j_g0ZxaQKpV6eB3G-dB5TgcXCrgEA,1424 +fastapi/datastructures.py,sha256=b2PEz77XGq-u3Ur1Inwk0AGjOsQZO49yF9C7IPJ15cY,5766 +fastapi/dependencies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fastapi/dependencies/__pycache__/__init__.cpython-310.pyc,, +fastapi/dependencies/__pycache__/models.cpython-310.pyc,, +fastapi/dependencies/__pycache__/utils.cpython-310.pyc,, +fastapi/dependencies/models.py,sha256=Pjl6vx-4nZ5Tta9kJa3-RfQKkXtCpS09-FhMgs9eWNs,1507 +fastapi/dependencies/utils.py,sha256=wGN-BAb0NpG-89nA_OllS0F4wYwGfhHgb8IuT3MTqck,36619 +fastapi/encoders.py,sha256=LvwYmFeOz4tVwvgBoC5rvZnbr7hZr73KGrU8O7zSptU,11068 +fastapi/exception_handlers.py,sha256=MBrIOA-ugjJDivIi4rSsUJBdTsjuzN76q4yh0q1COKw,1332 +fastapi/exceptions.py,sha256=taNixuFEXb67lI1bnX1ubq8y8TseJ4yoPlWjyP0fTzk,4969 +fastapi/logger.py,sha256=I9NNi3ov8AcqbsbC9wl1X-hdItKgYt2XTrx1f99Zpl4,54 +fastapi/middleware/__init__.py,sha256=oQDxiFVcc1fYJUOIFvphnK7pTT5kktmfL32QXpBFvvo,58 +fastapi/middleware/__pycache__/__init__.cpython-310.pyc,, +fastapi/middleware/__pycache__/cors.cpython-310.pyc,, +fastapi/middleware/__pycache__/gzip.cpython-310.pyc,, +fastapi/middleware/__pycache__/httpsredirect.cpython-310.pyc,, +fastapi/middleware/__pycache__/trustedhost.cpython-310.pyc,, +fastapi/middleware/__pycache__/wsgi.cpython-310.pyc,, +fastapi/middleware/cors.py,sha256=ynwjWQZoc_vbhzZ3_ZXceoaSrslHFHPdoM52rXr0WUU,79 +fastapi/middleware/gzip.py,sha256=xM5PcsH8QlAimZw4VDvcmTnqQamslThsfe3CVN2voa0,79 +fastapi/middleware/httpsredirect.py,sha256=rL8eXMnmLijwVkH7_400zHri1AekfeBd6D6qs8ix950,115 +fastapi/middleware/trustedhost.py,sha256=eE5XGRxGa7c5zPnMJDGp3BxaL25k5iVQlhnv-Pk0Pss,109 +fastapi/middleware/wsgi.py,sha256=Z3Ue-7wni4lUZMvH3G9ek__acgYdJstbnpZX_HQAboY,79 +fastapi/openapi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fastapi/openapi/__pycache__/__init__.cpython-310.pyc,, +fastapi/openapi/__pycache__/constants.cpython-310.pyc,, +fastapi/openapi/__pycache__/docs.cpython-310.pyc,, +fastapi/openapi/__pycache__/models.cpython-310.pyc,, +fastapi/openapi/__pycache__/utils.cpython-310.pyc,, +fastapi/openapi/constants.py,sha256=adGzmis1L1HJRTE3kJ5fmHS_Noq6tIY6pWv_SFzoFDU,153 +fastapi/openapi/docs.py,sha256=zSDv4xY6XHcKsaG4zyk1HqSnrZtfZFBB0J7ZBk5YHPE,10345 +fastapi/openapi/models.py,sha256=PqkxQiqcEgjKuhfUIWPZPQcyTcubtUCB3vcObLsB7VE,15397 +fastapi/openapi/utils.py,sha256=e00G_p0IdpiffBUaq31BUyiloXbpld8RryKYnYKisdY,23964 +fastapi/param_functions.py,sha256=JHNPLIYvoAwdnZZavIVsxOat8x23fX_Kl33reh7HKl8,64019 +fastapi/params.py,sha256=g450axUBQgQJODdtM7WBxZbQj9Z64inFvadrgHikBbU,28237 +fastapi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fastapi/requests.py,sha256=zayepKFcienBllv3snmWI20Gk0oHNVLU4DDhqXBb4LU,142 +fastapi/responses.py,sha256=QNQQlwpKhQoIPZTTWkpc9d_QGeGZ_aVQPaDV3nQ8m7c,1761 +fastapi/routing.py,sha256=-SaOgqaseKw5mlTCk-FliS6Wx5la_CjdV5FqSPDmW9g,176337 +fastapi/security/__init__.py,sha256=bO8pNmxqVRXUjfl2mOKiVZLn0FpBQ61VUYVjmppnbJw,881 +fastapi/security/__pycache__/__init__.cpython-310.pyc,, +fastapi/security/__pycache__/api_key.cpython-310.pyc,, +fastapi/security/__pycache__/base.cpython-310.pyc,, +fastapi/security/__pycache__/http.cpython-310.pyc,, +fastapi/security/__pycache__/oauth2.cpython-310.pyc,, +fastapi/security/__pycache__/open_id_connect_url.cpython-310.pyc,, +fastapi/security/__pycache__/utils.cpython-310.pyc,, +fastapi/security/api_key.py,sha256=cBI5Z4zWVjL1uJrsjTeLy7MafHPAO2HQPzTrpyoIYWA,9094 +fastapi/security/base.py,sha256=dl4pvbC-RxjfbWgPtCWd8MVU-7CB2SZ22rJDXVCXO6c,141 +fastapi/security/http.py,sha256=rWR2x-5CUsjWmRucYthwRig6MG1o-boyrr4Xo-PuuxU,13606 +fastapi/security/oauth2.py,sha256=M1AFIDT7G3oQChq83poI3eg8ZDeibcvnGmya2CTS7JY,22036 +fastapi/security/open_id_connect_url.py,sha256=8vizZ2tGqEp1ur8SwtVgyHJhGAJ5AqahgcvSpaIioDI,2722 +fastapi/security/utils.py,sha256=bd8T0YM7UQD5ATKucr1bNtAvz_Y3__dVNAv5UebiPvc,293 +fastapi/staticfiles.py,sha256=iirGIt3sdY2QZXd36ijs3Cj-T0FuGFda3cd90kM9Ikw,69 +fastapi/templating.py,sha256=4zsuTWgcjcEainMJFAlW6-gnslm6AgOS1SiiDWfmQxk,76 +fastapi/testclient.py,sha256=nBvaAmX66YldReJNZXPOk1sfuo2Q6hs8bOvIaCep6LQ,66 +fastapi/types.py,sha256=nFb36sK3DSoqoyo7Miwy3meKK5UdFBgkAgLSzQlUVyI,383 +fastapi/utils.py,sha256=y8Bj5ttMaI9tS4D60OUgXqKnktBr99NdYUnHHV9LgoY,7948 +fastapi/websockets.py,sha256=419uncYObEKZG0YcrXscfQQYLSWoE10jqxVMetGdR98,222 diff --git a/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/WHEEL b/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..045c8acdea31cbca5be986e915f784c1aafc720f --- /dev/null +++ b/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: pdm-backend (2.4.5) +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/entry_points.txt b/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..b81849e1ef82b793f6f81bb970cdc3cb791f5776 --- /dev/null +++ b/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/entry_points.txt @@ -0,0 +1,5 @@ +[console_scripts] +fastapi = fastapi.cli:main + +[gui_scripts] + diff --git a/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/licenses/LICENSE b/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..3e92463e6bd522a2a21e5f0a80d8089d6c4be20d --- /dev/null +++ b/lib/python3.1/site-packages/fastapi-0.116.1.dist-info/licenses/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Sebastián Ramírez + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/INSTALLER b/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/METADATA b/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..9e6e903e7da0936a5460fbde637fa45970b7196a --- /dev/null +++ b/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/METADATA @@ -0,0 +1,137 @@ +Metadata-Version: 2.1 +Name: fastapi-cli +Version: 0.0.8 +Summary: Run and manage FastAPI apps from the command line with FastAPI CLI. 🚀 +Author-Email: =?utf-8?q?Sebasti=C3=A1n_Ram=C3=ADrez?= +License: MIT +Classifier: Intended Audience :: Information Technology +Classifier: Intended Audience :: System Administrators +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python +Classifier: Topic :: Software Development :: Libraries :: Application Frameworks +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: Software Development +Classifier: Typing :: Typed +Classifier: Development Status :: 4 - Beta +Classifier: Framework :: FastAPI +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: License :: OSI Approved :: MIT License +Project-URL: Homepage, https://github.com/fastapi/fastapi-cli +Project-URL: Documentation, https://fastapi.tiangolo.com/fastapi-cli/ +Project-URL: Repository, https://github.com/fastapi/fastapi-cli +Project-URL: Issues, https://github.com/fastapi/fastapi-cli/issues +Project-URL: Changelog, https://github.com/fastapi/fastapi-cli/blob/main/release-notes.md +Requires-Python: >=3.8 +Requires-Dist: typer>=0.15.1 +Requires-Dist: uvicorn[standard]>=0.15.0 +Requires-Dist: rich-toolkit>=0.14.8 +Provides-Extra: standard +Requires-Dist: uvicorn[standard]>=0.15.0; extra == "standard" +Requires-Dist: fastapi-cloud-cli>=0.1.1; extra == "standard" +Provides-Extra: standard-no-fastapi-cloud-cli +Requires-Dist: uvicorn[standard]>=0.15.0; extra == "standard-no-fastapi-cloud-cli" +Description-Content-Type: text/markdown + +# FastAPI CLI + + + Test + + + Publish + + + Coverage + + Package version + + +--- + +**Source Code**: https://github.com/fastapi/fastapi-cli + +--- + +Run and manage FastAPI apps from the command line with FastAPI CLI. 🚀 + +## Description + +**FastAPI CLI** is a command line program `fastapi` that you can use to serve your FastAPI app, manage your FastAPI project, and more. + +When you install FastAPI (e.g. with `pip install "fastapi[standard]"`), it includes a package called `fastapi-cli`, this package provides the `fastapi` command in the terminal. + +To run your FastAPI app for development, you can use the `fastapi dev` command: + +
    + +```console +$ fastapi dev main.py + + FastAPI Starting development server 🚀 + + Searching for package file structure from directories with __init__.py files + Importing from /home/user/code/awesomeapp + + module 🐍 main.py + + code Importing the FastAPI app object from the module with the following code: + + from main import app + + app Using import string: main:app + + server Server started at http://127.0.0.1:8000 + server Documentation at http://127.0.0.1:8000/docs + + tip Running in development mode, for production use: fastapi run + + Logs: + + INFO Will watch for changes in these directories: ['/home/user/code/awesomeapp'] + INFO Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit) + INFO Started reloader process [4106097] using WatchFiles + INFO Started server process [4106120] + INFO Waiting for application startup. + INFO Application startup complete. +``` + +
    + +That command line program called `fastapi` is **FastAPI CLI**. + +FastAPI CLI takes the path to your Python program and automatically detects the variable with the FastAPI (commonly named `app`) and how to import it, and then serves it. + +For production you would use `fastapi run` instead. 🚀 + +Internally, **FastAPI CLI** uses Uvicorn, a high-performance, production-ready, ASGI server. 😎 + +## `fastapi dev` + +When you run `fastapi dev`, it will run on development mode. + +By default, it will have **auto-reload** enabled, so it will automatically reload the server when you make changes to your code. This is resource intensive and could be less stable than without it, you should only use it for development. + +By default it will listen on the IP address `127.0.0.1`, which is the IP for your machine to communicate with itself alone (`localhost`). + +## `fastapi run` + +When you run `fastapi run`, it will run on production mode by default. + +It will have **auto-reload disabled** by default. + +It will listen on the IP address `0.0.0.0`, which means all the available IP addresses, this way it will be publicly accessible to anyone that can communicate with the machine. This is how you would normally run it in production, for example, in a container. + +In most cases you would (and should) have a "termination proxy" handling HTTPS for you on top, this will depend on how you deploy your application, your provider might do this for you, or you might need to set it up yourself. You can learn more about it in the FastAPI Deployment documentation. + +## License + +This project is licensed under the terms of the MIT license. diff --git a/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/RECORD b/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..ae69156c6bbb80c6a3d0238a58faaff11e5ac56f --- /dev/null +++ b/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/RECORD @@ -0,0 +1,24 @@ +../../../bin/fastapi,sha256=CIgrT46-fisJx_WaGmfvQ3S3k7oy4z4foVDfvVEtjE8,248 +fastapi_cli-0.0.8.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +fastapi_cli-0.0.8.dist-info/METADATA,sha256=X79w3QC4o9tKdWy59HlE8P737JMvOQE8xx-aGzVuQqM,6342 +fastapi_cli-0.0.8.dist-info/RECORD,, +fastapi_cli-0.0.8.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90 +fastapi_cli-0.0.8.dist-info/entry_points.txt,sha256=L-dwDLGAhOlBVadq5cDBAB1i8y4oSFueX02A4gmMKco,65 +fastapi_cli-0.0.8.dist-info/licenses/LICENSE,sha256=FqD5B4VbXJnefprQseE0U8llL6FxojC-i8muZy7YmSU,1086 +fastapi_cli/__init__.py,sha256=wOJN3HxAgnSon5vWYU3Txm2UZ_7tBHDKXUKZIH-mXX8,22 +fastapi_cli/__main__.py,sha256=bYt9eEaoRQWdejEHFD8REx9jxVEdZptECFsV7F49Ink,30 +fastapi_cli/__pycache__/__init__.cpython-310.pyc,, +fastapi_cli/__pycache__/__main__.cpython-310.pyc,, +fastapi_cli/__pycache__/cli.cpython-310.pyc,, +fastapi_cli/__pycache__/discover.cpython-310.pyc,, +fastapi_cli/__pycache__/exceptions.cpython-310.pyc,, +fastapi_cli/__pycache__/logging.cpython-310.pyc,, +fastapi_cli/cli.py,sha256=c3_y-wlrzmJYlZPmZyFORYk_vEIHR2zgkny3ERcYwFk,11565 +fastapi_cli/discover.py,sha256=Q3CSEWt2V68JcNuAv31l7IcO_-146n2dBUSdtexJPYE,3971 +fastapi_cli/exceptions.py,sha256=AHRSqd43fbqN5IpX-Fq389k9MoEK_q28wVFL7oqPNcc,47 +fastapi_cli/logging.py,sha256=Yh2Nx5eC8XE_a3psTMO0kA5BM8lf63bBCqSMyDxUN7s,690 +fastapi_cli/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fastapi_cli/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fastapi_cli/utils/__pycache__/__init__.cpython-310.pyc,, +fastapi_cli/utils/__pycache__/cli.cpython-310.pyc,, +fastapi_cli/utils/cli.py,sha256=tAFRHnSurPgGX-JneQhGUJrzYLuuEKn1SxHK8Zj3nng,2268 diff --git a/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/WHEEL b/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..045c8acdea31cbca5be986e915f784c1aafc720f --- /dev/null +++ b/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: pdm-backend (2.4.5) +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/entry_points.txt b/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..a951c023691af7e76b4757aa8911ac2708bf85b6 --- /dev/null +++ b/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/entry_points.txt @@ -0,0 +1,5 @@ +[console_scripts] +fastapi = fastapi_cli.cli:main + +[gui_scripts] + diff --git a/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/licenses/LICENSE b/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..ef706eaeccc818c83d2921d7d1199f6ff9210a65 --- /dev/null +++ b/lib/python3.1/site-packages/fastapi_cli-0.0.8.dist-info/licenses/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Sebastián Ramírez + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/lib/python3.1/site-packages/fastapi_cli/__init__.py b/lib/python3.1/site-packages/fastapi_cli/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a73339bf81370efc24e667e80c0cd6d8b8b80687 --- /dev/null +++ b/lib/python3.1/site-packages/fastapi_cli/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.8" diff --git a/lib/python3.1/site-packages/fastapi_cli/__main__.py b/lib/python3.1/site-packages/fastapi_cli/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..4e28416e104515e90fca4b69cc60d0c61fd15d61 --- /dev/null +++ b/lib/python3.1/site-packages/fastapi_cli/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +main() diff --git a/lib/python3.1/site-packages/fastapi_cli/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/site-packages/fastapi_cli/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..971b5660efc714acf56ac4e370cfefe1edfc76b6 Binary files /dev/null and b/lib/python3.1/site-packages/fastapi_cli/__pycache__/__init__.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/fastapi_cli/__pycache__/__main__.cpython-310.pyc b/lib/python3.1/site-packages/fastapi_cli/__pycache__/__main__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cd8145d62b3d0a38a70fc2f169b9832c217cac92 Binary files /dev/null and b/lib/python3.1/site-packages/fastapi_cli/__pycache__/__main__.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/fastapi_cli/__pycache__/cli.cpython-310.pyc b/lib/python3.1/site-packages/fastapi_cli/__pycache__/cli.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f9558b66882a642e2405877473ef71cb9c7a3ee2 Binary files /dev/null and b/lib/python3.1/site-packages/fastapi_cli/__pycache__/cli.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/fastapi_cli/__pycache__/discover.cpython-310.pyc b/lib/python3.1/site-packages/fastapi_cli/__pycache__/discover.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..28748c177039a99b96cb900d5c5b5e70083b5683 Binary files /dev/null and b/lib/python3.1/site-packages/fastapi_cli/__pycache__/discover.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/fastapi_cli/__pycache__/exceptions.cpython-310.pyc b/lib/python3.1/site-packages/fastapi_cli/__pycache__/exceptions.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..393cb0770de12f04be8f9312922ba18e691a1345 Binary files /dev/null and b/lib/python3.1/site-packages/fastapi_cli/__pycache__/exceptions.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/fastapi_cli/__pycache__/logging.cpython-310.pyc b/lib/python3.1/site-packages/fastapi_cli/__pycache__/logging.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8da036d40ddfc5d084cfa6a378dbf585d04931b Binary files /dev/null and b/lib/python3.1/site-packages/fastapi_cli/__pycache__/logging.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/fastapi_cli/cli.py b/lib/python3.1/site-packages/fastapi_cli/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..77647217ec9928b98e595181bb023a80a58fa695 --- /dev/null +++ b/lib/python3.1/site-packages/fastapi_cli/cli.py @@ -0,0 +1,358 @@ +import logging +from pathlib import Path +from typing import Any, List, Union + +import typer +from rich import print +from rich.tree import Tree +from typing_extensions import Annotated + +from fastapi_cli.discover import get_import_data +from fastapi_cli.exceptions import FastAPICLIException + +from . import __version__ +from .logging import setup_logging +from .utils.cli import get_rich_toolkit, get_uvicorn_log_config + +app = typer.Typer(rich_markup_mode="rich") + +logger = logging.getLogger(__name__) + + +try: + import uvicorn +except ImportError: # pragma: no cover + uvicorn = None # type: ignore[assignment] + + +try: + from fastapi_cloud_cli.cli import ( + app as fastapi_cloud_cli, + ) + + app.add_typer(fastapi_cloud_cli) +except ImportError: # pragma: no cover + pass + + +def version_callback(value: bool) -> None: + if value: + print(f"FastAPI CLI version: [green]{__version__}[/green]") + raise typer.Exit() + + +@app.callback() +def callback( + version: Annotated[ + Union[bool, None], + typer.Option( + "--version", help="Show the version and exit.", callback=version_callback + ), + ] = None, + verbose: bool = typer.Option(False, help="Enable verbose output"), +) -> None: + """ + FastAPI CLI - The [bold]fastapi[/bold] command line app. 😎 + + Manage your [bold]FastAPI[/bold] projects, run your FastAPI apps, and more. + + Read more in the docs: [link=https://fastapi.tiangolo.com/fastapi-cli/]https://fastapi.tiangolo.com/fastapi-cli/[/link]. + """ + + log_level = logging.DEBUG if verbose else logging.INFO + + setup_logging(level=log_level) + + +def _get_module_tree(module_paths: List[Path]) -> Tree: + root = module_paths[0] + name = f"🐍 {root.name}" if root.is_file() else f"📁 {root.name}" + + root_tree = Tree(name) + + if root.is_dir(): + root_tree.add("[dim]🐍 __init__.py[/dim]") + + tree = root_tree + for sub_path in module_paths[1:]: + sub_name = ( + f"🐍 {sub_path.name}" if sub_path.is_file() else f"📁 {sub_path.name}" + ) + tree = tree.add(sub_name) + if sub_path.is_dir(): + tree.add("[dim]🐍 __init__.py[/dim]") + + return root_tree + + +def _run( + path: Union[Path, None] = None, + *, + host: str = "127.0.0.1", + port: int = 8000, + reload: bool = True, + workers: Union[int, None] = None, + root_path: str = "", + command: str, + app: Union[str, None] = None, + proxy_headers: bool = False, +) -> None: + with get_rich_toolkit() as toolkit: + server_type = "development" if command == "dev" else "production" + + toolkit.print_title(f"Starting {server_type} server 🚀", tag="FastAPI") + toolkit.print_line() + + toolkit.print( + "Searching for package file structure from directories with [blue]__init__.py[/blue] files" + ) + + try: + import_data = get_import_data(path=path, app_name=app) + except FastAPICLIException as e: + toolkit.print_line() + toolkit.print(f"[error]{e}") + raise typer.Exit(code=1) from None + + logger.debug(f"Importing from {import_data.module_data.extra_sys_path}") + logger.debug(f"Importing module {import_data.module_data.module_import_str}") + + module_data = import_data.module_data + import_string = import_data.import_string + + toolkit.print(f"Importing from {module_data.extra_sys_path}") + toolkit.print_line() + + root_tree = _get_module_tree(module_data.module_paths) + + toolkit.print(root_tree, tag="module") + toolkit.print_line() + + toolkit.print( + "Importing the FastAPI app object from the module with the following code:", + tag="code", + ) + toolkit.print_line() + toolkit.print( + f"[underline]from [bold]{module_data.module_import_str}[/bold] import [bold]{import_data.app_name}[/bold]" + ) + toolkit.print_line() + + toolkit.print( + f"Using import string: [blue]{import_string}[/]", + tag="app", + ) + + url = f"http://{host}:{port}" + url_docs = f"{url}/docs" + + toolkit.print_line() + toolkit.print( + f"Server started at [link={url}]{url}[/]", + f"Documentation at [link={url_docs}]{url_docs}[/]", + tag="server", + ) + + if command == "dev": + toolkit.print_line() + toolkit.print( + "Running in development mode, for production use: [bold]fastapi run[/]", + tag="tip", + ) + + if not uvicorn: + raise FastAPICLIException( + "Could not import Uvicorn, try running 'pip install uvicorn'" + ) from None + + toolkit.print_line() + toolkit.print("Logs:") + toolkit.print_line() + + uvicorn.run( + app=import_string, + host=host, + port=port, + reload=reload, + workers=workers, + root_path=root_path, + proxy_headers=proxy_headers, + log_config=get_uvicorn_log_config(), + ) + + +@app.command() +def dev( + path: Annotated[ + Union[Path, None], + typer.Argument( + help="A path to a Python file or package directory (with [blue]__init__.py[/blue] files) containing a [bold]FastAPI[/bold] app. If not provided, a default set of paths will be tried." + ), + ] = None, + *, + host: Annotated[ + str, + typer.Option( + help="The host to serve on. For local development in localhost use [blue]127.0.0.1[/blue]. To enable public access, e.g. in a container, use all the IP addresses available with [blue]0.0.0.0[/blue]." + ), + ] = "127.0.0.1", + port: Annotated[ + int, + typer.Option( + help="The port to serve on. You would normally have a termination proxy on top (another program) handling HTTPS on port [blue]443[/blue] and HTTP on port [blue]80[/blue], transferring the communication to your app." + ), + ] = 8000, + reload: Annotated[ + bool, + typer.Option( + help="Enable auto-reload of the server when (code) files change. This is [bold]resource intensive[/bold], use it only during development." + ), + ] = True, + root_path: Annotated[ + str, + typer.Option( + help="The root path is used to tell your app that it is being served to the outside world with some [bold]path prefix[/bold] set up in some termination proxy or similar." + ), + ] = "", + app: Annotated[ + Union[str, None], + typer.Option( + help="The name of the variable that contains the [bold]FastAPI[/bold] app in the imported module or package. If not provided, it is detected automatically." + ), + ] = None, + proxy_headers: Annotated[ + bool, + typer.Option( + help="Enable/Disable X-Forwarded-Proto, X-Forwarded-For, X-Forwarded-Port to populate remote address info." + ), + ] = True, +) -> Any: + """ + Run a [bold]FastAPI[/bold] app in [yellow]development[/yellow] mode. 🧪 + + This is equivalent to [bold]fastapi run[/bold] but with [bold]reload[/bold] enabled and listening on the [blue]127.0.0.1[/blue] address. + + It automatically detects the Python module or package that needs to be imported based on the file or directory path passed. + + If no path is passed, it tries with: + + - [blue]main.py[/blue] + - [blue]app.py[/blue] + - [blue]api.py[/blue] + - [blue]app/main.py[/blue] + - [blue]app/app.py[/blue] + - [blue]app/api.py[/blue] + + It also detects the directory that needs to be added to the [bold]PYTHONPATH[/bold] to make the app importable and adds it. + + It detects the [bold]FastAPI[/bold] app object to use. By default it looks in the module or package for an object named: + + - [blue]app[/blue] + - [blue]api[/blue] + + Otherwise, it uses the first [bold]FastAPI[/bold] app found in the imported module or package. + """ + _run( + path=path, + host=host, + port=port, + reload=reload, + root_path=root_path, + app=app, + command="dev", + proxy_headers=proxy_headers, + ) + + +@app.command() +def run( + path: Annotated[ + Union[Path, None], + typer.Argument( + help="A path to a Python file or package directory (with [blue]__init__.py[/blue] files) containing a [bold]FastAPI[/bold] app. If not provided, a default set of paths will be tried." + ), + ] = None, + *, + host: Annotated[ + str, + typer.Option( + help="The host to serve on. For local development in localhost use [blue]127.0.0.1[/blue]. To enable public access, e.g. in a container, use all the IP addresses available with [blue]0.0.0.0[/blue]." + ), + ] = "0.0.0.0", + port: Annotated[ + int, + typer.Option( + help="The port to serve on. You would normally have a termination proxy on top (another program) handling HTTPS on port [blue]443[/blue] and HTTP on port [blue]80[/blue], transferring the communication to your app." + ), + ] = 8000, + reload: Annotated[ + bool, + typer.Option( + help="Enable auto-reload of the server when (code) files change. This is [bold]resource intensive[/bold], use it only during development." + ), + ] = False, + workers: Annotated[ + Union[int, None], + typer.Option( + help="Use multiple worker processes. Mutually exclusive with the --reload flag." + ), + ] = None, + root_path: Annotated[ + str, + typer.Option( + help="The root path is used to tell your app that it is being served to the outside world with some [bold]path prefix[/bold] set up in some termination proxy or similar." + ), + ] = "", + app: Annotated[ + Union[str, None], + typer.Option( + help="The name of the variable that contains the [bold]FastAPI[/bold] app in the imported module or package. If not provided, it is detected automatically." + ), + ] = None, + proxy_headers: Annotated[ + bool, + typer.Option( + help="Enable/Disable X-Forwarded-Proto, X-Forwarded-For, X-Forwarded-Port to populate remote address info." + ), + ] = True, +) -> Any: + """ + Run a [bold]FastAPI[/bold] app in [green]production[/green] mode. 🚀 + + This is equivalent to [bold]fastapi dev[/bold] but with [bold]reload[/bold] disabled and listening on the [blue]0.0.0.0[/blue] address. + + It automatically detects the Python module or package that needs to be imported based on the file or directory path passed. + + If no path is passed, it tries with: + + - [blue]main.py[/blue] + - [blue]app.py[/blue] + - [blue]api.py[/blue] + - [blue]app/main.py[/blue] + - [blue]app/app.py[/blue] + - [blue]app/api.py[/blue] + + It also detects the directory that needs to be added to the [bold]PYTHONPATH[/bold] to make the app importable and adds it. + + It detects the [bold]FastAPI[/bold] app object to use. By default it looks in the module or package for an object named: + + - [blue]app[/blue] + - [blue]api[/blue] + + Otherwise, it uses the first [bold]FastAPI[/bold] app found in the imported module or package. + """ + _run( + path=path, + host=host, + port=port, + reload=reload, + workers=workers, + root_path=root_path, + app=app, + command="run", + proxy_headers=proxy_headers, + ) + + +def main() -> None: + app() diff --git a/lib/python3.1/site-packages/fastapi_cli/discover.py b/lib/python3.1/site-packages/fastapi_cli/discover.py new file mode 100644 index 0000000000000000000000000000000000000000..43d0e9c976d7026fd65b14901a53bc3ccd792bf1 --- /dev/null +++ b/lib/python3.1/site-packages/fastapi_cli/discover.py @@ -0,0 +1,132 @@ +import importlib +import sys +from dataclasses import dataclass +from logging import getLogger +from pathlib import Path +from typing import List, Union + +from fastapi_cli.exceptions import FastAPICLIException + +logger = getLogger(__name__) + +try: + from fastapi import FastAPI +except ImportError: # pragma: no cover + FastAPI = None # type: ignore[misc, assignment] + + +def get_default_path() -> Path: + potential_paths = ( + "main.py", + "app.py", + "api.py", + "app/main.py", + "app/app.py", + "app/api.py", + ) + + for full_path in potential_paths: + path = Path(full_path) + if path.is_file(): + return path + + raise FastAPICLIException( + "Could not find a default file to run, please provide an explicit path" + ) + + +@dataclass +class ModuleData: + module_import_str: str + extra_sys_path: Path + module_paths: List[Path] + + +def get_module_data_from_path(path: Path) -> ModuleData: + use_path = path.resolve() + module_path = use_path + if use_path.is_file() and use_path.stem == "__init__": + module_path = use_path.parent + module_paths = [module_path] + extra_sys_path = module_path.parent + for parent in module_path.parents: + init_path = parent / "__init__.py" + if init_path.is_file(): + module_paths.insert(0, parent) + extra_sys_path = parent.parent + else: + break + + module_str = ".".join(p.stem for p in module_paths) + return ModuleData( + module_import_str=module_str, + extra_sys_path=extra_sys_path.resolve(), + module_paths=module_paths, + ) + + +def get_app_name(*, mod_data: ModuleData, app_name: Union[str, None] = None) -> str: + try: + mod = importlib.import_module(mod_data.module_import_str) + except (ImportError, ValueError) as e: + logger.error(f"Import error: {e}") + logger.warning( + "Ensure all the package directories have an [blue]__init__.py[/blue] file" + ) + raise + if not FastAPI: # type: ignore[truthy-function] + raise FastAPICLIException( + "Could not import FastAPI, try running 'pip install fastapi'" + ) from None + object_names = dir(mod) + object_names_set = set(object_names) + if app_name: + if app_name not in object_names_set: + raise FastAPICLIException( + f"Could not find app name {app_name} in {mod_data.module_import_str}" + ) + app = getattr(mod, app_name) + if not isinstance(app, FastAPI): + raise FastAPICLIException( + f"The app name {app_name} in {mod_data.module_import_str} doesn't seem to be a FastAPI app" + ) + return app_name + for preferred_name in ["app", "api"]: + if preferred_name in object_names_set: + obj = getattr(mod, preferred_name) + if isinstance(obj, FastAPI): + return preferred_name + for name in object_names: + obj = getattr(mod, name) + if isinstance(obj, FastAPI): + return name + raise FastAPICLIException("Could not find FastAPI app in module, try using --app") + + +@dataclass +class ImportData: + app_name: str + module_data: ModuleData + import_string: str + + +def get_import_data( + *, path: Union[Path, None] = None, app_name: Union[str, None] = None +) -> ImportData: + if not path: + path = get_default_path() + + logger.debug(f"Using path [blue]{path}[/blue]") + logger.debug(f"Resolved absolute path {path.resolve()}") + + if not path.exists(): + raise FastAPICLIException(f"Path does not exist {path}") + mod_data = get_module_data_from_path(path) + sys.path.insert(0, str(mod_data.extra_sys_path)) + use_app_name = get_app_name(mod_data=mod_data, app_name=app_name) + + import_string = f"{mod_data.module_import_str}:{use_app_name}" + + return ImportData( + app_name=use_app_name, module_data=mod_data, import_string=import_string + ) diff --git a/lib/python3.1/site-packages/fastapi_cli/exceptions.py b/lib/python3.1/site-packages/fastapi_cli/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..a3d9a05615d99183e246eae126a0a0b456e5c30c --- /dev/null +++ b/lib/python3.1/site-packages/fastapi_cli/exceptions.py @@ -0,0 +1,2 @@ +class FastAPICLIException(Exception): + pass diff --git a/lib/python3.1/site-packages/fastapi_cli/logging.py b/lib/python3.1/site-packages/fastapi_cli/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..67f116c97eb0967c6e867b8fa6ed18b57619d007 --- /dev/null +++ b/lib/python3.1/site-packages/fastapi_cli/logging.py @@ -0,0 +1,25 @@ +import logging +from typing import Union + +from rich.console import Console +from rich.logging import RichHandler + + +def setup_logging( + terminal_width: Union[int, None] = None, level: int = logging.INFO +) -> None: + logger = logging.getLogger("fastapi_cli") + console = Console(width=terminal_width) if terminal_width else None + rich_handler = RichHandler( + show_time=False, + rich_tracebacks=True, + tracebacks_show_locals=True, + markup=True, + show_path=False, + console=console, + ) + rich_handler.setFormatter(logging.Formatter("%(message)s")) + logger.addHandler(rich_handler) + + logger.setLevel(level) + logger.propagate = False diff --git a/lib/python3.1/site-packages/fastapi_cli/py.typed b/lib/python3.1/site-packages/fastapi_cli/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.1/site-packages/fastapi_cli/utils/__init__.py b/lib/python3.1/site-packages/fastapi_cli/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.1/site-packages/fastapi_cli/utils/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/site-packages/fastapi_cli/utils/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65fcecf89a680ab0eaeb447e79b4248d41cadb88 Binary files /dev/null and b/lib/python3.1/site-packages/fastapi_cli/utils/__pycache__/__init__.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/fastapi_cli/utils/__pycache__/cli.cpython-310.pyc b/lib/python3.1/site-packages/fastapi_cli/utils/__pycache__/cli.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a91cc16e71847b50043c515eb55410773a16d97d Binary files /dev/null and b/lib/python3.1/site-packages/fastapi_cli/utils/__pycache__/cli.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/fastapi_cli/utils/cli.py b/lib/python3.1/site-packages/fastapi_cli/utils/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..f6805e7c6cf7bd3d57ab55f5030c0a98d75092ca --- /dev/null +++ b/lib/python3.1/site-packages/fastapi_cli/utils/cli.py @@ -0,0 +1,76 @@ +import logging +from typing import Any, Dict + +from rich_toolkit import RichToolkit, RichToolkitTheme +from rich_toolkit.styles import TaggedStyle +from uvicorn.logging import DefaultFormatter + + +class CustomFormatter(DefaultFormatter): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.toolkit = get_rich_toolkit() + + def formatMessage(self, record: logging.LogRecord) -> str: + return self.toolkit.print_as_string(record.getMessage(), tag=record.levelname) + + +def get_uvicorn_log_config() -> Dict[str, Any]: + return { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "default": { + "()": CustomFormatter, + "fmt": "%(levelprefix)s %(message)s", + "use_colors": None, + }, + "access": { + "()": CustomFormatter, + "fmt": "%(levelprefix)s %(client_addr)s - '%(request_line)s' %(status_code)s", + }, + }, + "handlers": { + "default": { + "formatter": "default", + "class": "logging.StreamHandler", + "stream": "ext://sys.stderr", + }, + "access": { + "formatter": "access", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", + }, + }, + "loggers": { + "uvicorn": {"handlers": ["default"], "level": "INFO"}, + "uvicorn.error": {"level": "INFO"}, + "uvicorn.access": { + "handlers": ["access"], + "level": "INFO", + "propagate": False, + }, + }, + } + + +logger = logging.getLogger(__name__) + + +def get_rich_toolkit() -> RichToolkit: + theme = RichToolkitTheme( + style=TaggedStyle(tag_width=11), + theme={ + "tag.title": "white on #009485", + "tag": "white on #007166", + "placeholder": "grey85", + "text": "white", + "selected": "#007166", + "result": "grey85", + "progress": "on #007166", + "error": "red", + "log.info": "black on blue", + }, + ) + + return RichToolkit(theme=theme) diff --git a/lib/python3.1/site-packages/gradio_client-1.12.1.dist-info/INSTALLER b/lib/python3.1/site-packages/gradio_client-1.12.1.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.1/site-packages/gradio_client-1.12.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.1/site-packages/gradio_client-1.12.1.dist-info/METADATA b/lib/python3.1/site-packages/gradio_client-1.12.1.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..59d9691b0434137d086f6e9f51dda08295f3bbaf --- /dev/null +++ b/lib/python3.1/site-packages/gradio_client-1.12.1.dist-info/METADATA @@ -0,0 +1,172 @@ +Metadata-Version: 2.4 +Name: gradio_client +Version: 1.12.1 +Summary: Python library for easily interacting with trained machine learning models +Project-URL: Homepage, https://github.com/gradio-app/gradio +Author-email: Abubakar Abid , Ali Abid , Ali Abdalla , Dawood Khan , Ahsen Khaliq , Pete Allen , Freddy Boulton +License-Expression: Apache-2.0 +Keywords: API,client,machine learning +Classifier: Development Status :: 4 - Beta +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Topic :: Scientific/Engineering +Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence +Classifier: Topic :: Software Development :: User Interfaces +Requires-Python: >=3.10 +Requires-Dist: fsspec +Requires-Dist: httpx>=0.24.1 +Requires-Dist: huggingface-hub>=0.19.3 +Requires-Dist: packaging +Requires-Dist: typing-extensions~=4.0 +Requires-Dist: websockets<16.0,>=10.0 +Description-Content-Type: text/markdown + +# `gradio_client`: Use a Gradio app as an API -- in 3 lines of Python + +This directory contains the source code for `gradio_client`, a lightweight Python library that makes it very easy to use any Gradio app as an API. + +As an example, consider this [Hugging Face Space that transcribes audio files](https://huggingface.co/spaces/abidlabs/whisper) that are recorded from the microphone. + +![](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/gradio-guides/whisper-screenshot.jpg) + +Using the `gradio_client` library, we can easily use the Gradio as an API to transcribe audio files programmatically. + +Here's the entire code to do it: + +```python +from gradio_client import Client + +client = Client("abidlabs/whisper") +client.predict("audio_sample.wav") + +>> "This is a test of the whisper speech recognition model." +``` + +The Gradio client works with any Gradio Space, whether it be an image generator, a stateful chatbot, or a tax calculator. + +## Installation + +If you already have a recent version of `gradio`, then the `gradio_client` is included as a dependency. + +Otherwise, the lightweight `gradio_client` package can be installed from pip (or pip3) and works with Python versions 3.10 or higher: + +```bash +$ pip install gradio_client +``` + +## Basic Usage + +### Connecting to a Space or a Gradio app + +Start by connecting instantiating a `Client` object and connecting it to a Gradio app that is running on Spaces (or anywhere else)! + +**Connecting to a Space** + +```python +from gradio_client import Client + +client = Client("abidlabs/en2fr") # a Space that translates from English to French +``` + +You can also connect to private Spaces by passing in your HF token with the `hf_token` parameter. You can get your HF token here: https://huggingface.co/settings/tokens + +```python +from gradio_client import Client + +client = Client("abidlabs/my-private-space", hf_token="...") +``` + +**Duplicating a Space for private use** + +While you can use any public Space as an API, you may get rate limited by Hugging Face if you make too many requests. For unlimited usage of a Space, simply duplicate the Space to create a private Space, +and then use it to make as many requests as you'd like! + +The `gradio_client` includes a class method: `Client.duplicate()` to make this process simple: + +```python +from gradio_client import Client + +client = Client.duplicate("abidlabs/whisper") +client.predict("audio_sample.wav") + +>> "This is a test of the whisper speech recognition model." +``` + +If you have previously duplicated a Space, re-running `duplicate()` will _not_ create a new Space. Instead, the Client will attach to the previously-created Space. So it is safe to re-run the `Client.duplicate()` method multiple times. + +**Note:** if the original Space uses GPUs, your private Space will as well, and your Hugging Face account will get billed based on the price of the GPU. To minimize charges, your Space will automatically go to sleep after 1 hour of inactivity. You can also set the hardware using the `hardware` parameter of `duplicate()`. + +**Connecting a general Gradio app** + +If your app is running somewhere else, just provide the full URL instead, including the "http://" or "https://". Here's an example of making predictions to a Gradio app that is running on a share URL: + +```python +from gradio_client import Client + +client = Client("https://bec81a83-5b5c-471e.gradio.live") +``` + +### Inspecting the API endpoints + +Once you have connected to a Gradio app, you can view the APIs that are available to you by calling the `.view_api()` method. For the Whisper Space, we see the following: + +``` +Client.predict() Usage Info +--------------------------- +Named API endpoints: 1 + + - predict(input_audio, api_name="/predict") -> value_0 + Parameters: + - [Audio] input_audio: str (filepath or URL) + Returns: + - [Textbox] value_0: str (value) +``` + +This shows us that we have 1 API endpoint in this space, and shows us how to use the API endpoint to make a prediction: we should call the `.predict()` method, providing a parameter `input_audio` of type `str`, which is a `filepath or URL`. + +We should also provide the `api_name='/predict'` argument. Although this isn't necessary if a Gradio app has a single named endpoint, it does allow us to call different endpoints in a single app if they are available. If an app has unnamed API endpoints, these can also be displayed by running `.view_api(all_endpoints=True)`. + +### Making a prediction + +The simplest way to make a prediction is simply to call the `.predict()` function with the appropriate arguments: + +```python +from gradio_client import Client + +client = Client("abidlabs/en2fr") +client.predict("Hello") + +>> Bonjour +``` + +If there are multiple parameters, then you should pass them as separate arguments to `.predict()`, like this: + +```python +from gradio_client import Client + +client = Client("gradio/calculator") +client.predict(4, "add", 5) + +>> 9.0 +``` + +For certain inputs, such as images, you should pass in the filepath or URL to the file. Likewise, for the corresponding output types, you will get a filepath or URL returned. + +```python +from gradio_client import Client + +client = Client("abidlabs/whisper") +client.predict("https://audio-samples.github.io/samples/mp3/blizzard_unconditional/sample-0.mp3") + +>> "My thought I have nobody by a beauty and will as you poured. Mr. Rochester is serve in that so don't find simpus, and devoted abode, to at might in a r—" +``` + +## Advanced Usage + +For more ways to use the Gradio Python Client, check out our dedicated Guide on the Python client, available here: https://www.gradio.app/guides/getting-started-with-the-python-client diff --git a/lib/python3.1/site-packages/gradio_client-1.12.1.dist-info/RECORD b/lib/python3.1/site-packages/gradio_client-1.12.1.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..670731369207798aa7999ea21ecd38f52027213d --- /dev/null +++ b/lib/python3.1/site-packages/gradio_client-1.12.1.dist-info/RECORD @@ -0,0 +1,31 @@ +gradio_client-1.12.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +gradio_client-1.12.1.dist-info/METADATA,sha256=N1VpdoLfbJRgoCztAJCPHIfLXzMUzJL9N5xGyQvuZVw,7116 +gradio_client-1.12.1.dist-info/RECORD,, +gradio_client-1.12.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87 +gradio_client/CHANGELOG.md,sha256=Y3W9CzLecz1WnEbOv9wym8imholhOHqCq6o0ZZsVwbc,51472 +gradio_client/__init__.py,sha256=GtVHNFpTWZo0hL7CZTjwGiP3Ml2iHQyDof25LhmxSd0,246 +gradio_client/__pycache__/__init__.cpython-310.pyc,, +gradio_client/__pycache__/client.cpython-310.pyc,, +gradio_client/__pycache__/compatibility.cpython-310.pyc,, +gradio_client/__pycache__/data_classes.cpython-310.pyc,, +gradio_client/__pycache__/documentation.cpython-310.pyc,, +gradio_client/__pycache__/exceptions.cpython-310.pyc,, +gradio_client/__pycache__/media_data.cpython-310.pyc,, +gradio_client/__pycache__/serializing.cpython-310.pyc,, +gradio_client/__pycache__/utils.cpython-310.pyc,, +gradio_client/cli/__init__.py,sha256=jW4mYjLdQ-UbPqKNMKmhkNLKu2lBU2pDn4gTKPX-0bg,75 +gradio_client/cli/__pycache__/__init__.cpython-310.pyc,, +gradio_client/cli/__pycache__/deploy_discord.cpython-310.pyc,, +gradio_client/cli/deploy_discord.py,sha256=-5nblBN2bU7mHaLwLbE6p4OiuCLQbodRL4woA2Y_Rks,1325 +gradio_client/client.py,sha256=RgEM-HR6bm8XMvo5Pp4EL9E80jX7cAkAvCua1i1TZCk,74092 +gradio_client/compatibility.py,sha256=fVh3kn6K1J3rhJdt50YJ1gR-gmL04maH4r_kCBn_vOM,13940 +gradio_client/data_classes.py,sha256=tMdBJBjvdqqwg4XSgXpSH7sVHnCfxQY48QuSGf8nYN4,735 +gradio_client/documentation.py,sha256=_SXy06fCcF9aRUo163exKoTz-KqwM3tpgqgiQSQnsy4,14046 +gradio_client/exceptions.py,sha256=rgHdrWypLWpxfJXuJue5lUwqUmIj9AVdSYAwNDdGvE0,1440 +gradio_client/media_data.py,sha256=GQZQJbrbj3dE3_TIUcmDOFmp8DbuBkP7DQTxYT6NN3Y,722623 +gradio_client/package.json,sha256=unr4xFAav7qXus8dU1nNR_Pqtd4oPMCiRqsI6wJz9P8,133 +gradio_client/serializing.py,sha256=gg-yd_DDGpreDYZwrmJqNBL8e8UCXD81wAsu4orAjDg,21045 +gradio_client/templates/__pycache__/discord_chat.cpython-310.pyc,, +gradio_client/templates/discord_chat.py,sha256=KEaR8ldH6oSMUMWkULyUKVub-pSevBJ2VeU2bI-710o,5610 +gradio_client/types.json,sha256=hTBJNs5ZyXC1-f-0UfzFezdhzNp3OnUhNs3Qv-muAQ0,4432 +gradio_client/utils.py,sha256=rVuUuU7njzd6uoDmVnpNlQ9xAr4eVCU2jplLOyJtAls,44376 diff --git a/lib/python3.1/site-packages/gradio_client-1.12.1.dist-info/WHEEL b/lib/python3.1/site-packages/gradio_client-1.12.1.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..12228d414b6cfed7c39d3781c85c63256a1d7fb5 --- /dev/null +++ b/lib/python3.1/site-packages/gradio_client-1.12.1.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.27.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/lib/python3.1/site-packages/hjson-3.1.0.dist-info/INSTALLER b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.1/site-packages/hjson-3.1.0.dist-info/LICENSE.txt b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..2457de3c33ed058a53dee761ffb864fc3f2bc658 --- /dev/null +++ b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/LICENSE.txt @@ -0,0 +1,81 @@ +hjson (based on simplejson) is dual-licensed software. It is available under the terms +of the MIT license, or the Academic Free License version 2.1. The full +text of each license agreement is included below. This code is also +licensed to the Python Software Foundation (PSF) under a Contributor +Agreement. + +MIT License +=========== + +Copyright (c) 2006 Bob Ippolito +Copyright (c) 2015 Christian Zangl + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Academic Free License v. 2.1 +============================ + +Copyright (c) 2006 Bob Ippolito. All rights reserved. +Copyright (c) 2015 Christian Zangl + +This Academic Free License (the "License") applies to any original work of authorship (the "Original Work") whose owner (the "Licensor") has placed the following notice immediately following the copyright notice for the Original Work: + +Licensed under the Academic Free License version 2.1 + +1) Grant of Copyright License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license to do the following: + +a) to reproduce the Original Work in copies; + +b) to prepare derivative works ("Derivative Works") based upon the Original Work; + +c) to distribute copies of the Original Work and Derivative Works to the public; + +d) to perform the Original Work publicly; and + +e) to display the Original Work publicly. + +2) Grant of Patent License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license, under patent claims owned or controlled by the Licensor that are embodied in the Original Work as furnished by the Licensor, to make, use, sell and offer for sale the Original Work and Derivative Works. + +3) Grant of Source Code License. The term "Source Code" means the preferred form of the Original Work for making modifications to it and all available documentation describing how to modify the Original Work. Licensor hereby agrees to provide a machine-readable copy of the Source Code of the Original Work along with each copy of the Original Work that Licensor distributes. Licensor reserves the right to satisfy this obligation by placing a machine-readable copy of the Source Code in an information repository reasonably calculated to permit inexpensive and convenient access by You for as long as Licensor continues to distribute the Original Work, and by publishing the address of that information repository in a notice immediately following the copyright notice that applies to the Original Work. + +4) Exclusions From License Grant. Neither the names of Licensor, nor the names of any contributors to the Original Work, nor any of their trademarks or service marks, may be used to endorse or promote products derived from this Original Work without express prior written permission of the Licensor. Nothing in this License shall be deemed to grant any rights to trademarks, copyrights, patents, trade secrets or any other intellectual property of Licensor except as expressly stated herein. No patent license is granted to make, use, sell or offer to sell embodiments of any patent claims other than the licensed claims defined in Section 2. No right is granted to the trademarks of Licensor even if such marks are included in the Original Work. Nothing in this License shall be interpreted to prohibit Licensor from licensing under different terms from this License any Original Work that Licensor otherwise would have a right to license. + +5) This section intentionally omitted. + +6) Attribution Rights. You must retain, in the Source Code of any Derivative Works that You create, all copyright, patent or trademark notices from the Source Code of the Original Work, as well as any notices of licensing and any descriptive text identified therein as an "Attribution Notice." You must cause the Source Code for any Derivative Works that You create to carry a prominent Attribution Notice reasonably calculated to inform recipients that You have modified the Original Work. + +7) Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that the copyright in and to the Original Work and the patent rights granted herein by Licensor are owned by the Licensor or are sublicensed to You under the terms of this License with the permission of the contributor(s) of those copyrights and patent rights. Except as expressly stated in the immediately proceeding sentence, the Original Work is provided under this License on an "AS IS" BASIS and WITHOUT WARRANTY, either express or implied, including, without limitation, the warranties of NON-INFRINGEMENT, MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an essential part of this License. No license to Original Work is granted hereunder except under this disclaimer. + +8) Limitation of Liability. Under no circumstances and under no legal theory, whether in tort (including negligence), contract, or otherwise, shall the Licensor be liable to any person for any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or the use of the Original Work including, without limitation, damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses. This limitation of liability shall not apply to liability for death or personal injury resulting from Licensor's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. + +9) Acceptance and Termination. If You distribute copies of the Original Work or a Derivative Work, You must make a reasonable effort under the circumstances to obtain the express assent of recipients to the terms of this License. Nothing else but this License (or another written agreement between Licensor and You) grants You permission to create Derivative Works based upon the Original Work or to exercise any of the rights granted in Section 1 herein, and any attempt to do so except under the terms of this License (or another written agreement between Licensor and You) is expressly prohibited by U.S. copyright law, the equivalent laws of other countries, and by international treaty. Therefore, by exercising any of the rights granted to You in Section 1 herein, You indicate Your acceptance of this License and all of its terms and conditions. + +10) Termination for Patent Action. This License shall terminate automatically and You may no longer exercise any of the rights granted to You by this License as of the date You commence an action, including a cross-claim or counterclaim, against Licensor or any licensee alleging that the Original Work infringes a patent. This termination provision shall not apply for an action alleging patent infringement by combinations of the Original Work with other software or hardware. + +11) Jurisdiction, Venue and Governing Law. Any action or suit relating to this License may be brought only in the courts of a jurisdiction wherein the Licensor resides or in which Licensor conducts its primary business, and under the laws of that jurisdiction excluding its conflict-of-law provisions. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any use of the Original Work outside the scope of this License or after its termination shall be subject to the requirements and penalties of the U.S. Copyright Act, 17 U.S.C. § 101 et seq., the equivalent laws of other countries, and international treaty. This section shall survive the termination of this License. + +12) Attorneys Fees. In any action to enforce the terms of this License or seeking damages relating thereto, the prevailing party shall be entitled to recover its costs and expenses, including, without limitation, reasonable attorneys' fees and costs incurred in connection with such action, including any appeal of such action. This section shall survive the termination of this License. + +13) Miscellaneous. This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. + +14) Definition of "You" in This License. "You" throughout this License, whether in upper or lower case, means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with you. For purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +15) Right to Use. You may use the Original Work in all ways not otherwise restricted or conditioned by this License or by law, and Licensor promises not to interfere with or be responsible for such uses by You. + +This license is Copyright (C) 2003-2004 Lawrence E. Rosen. All rights reserved. Permission is hereby granted to copy and distribute this license without modification. This license may not be modified without the express written permission of its copyright owner. diff --git a/lib/python3.1/site-packages/hjson-3.1.0.dist-info/METADATA b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..5d78e45f7013def95c59fe264fd5cf54c1f00b31 --- /dev/null +++ b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/METADATA @@ -0,0 +1,125 @@ +Metadata-Version: 2.1 +Name: hjson +Version: 3.1.0 +Summary: Hjson, a user interface for JSON. +Home-page: http://github.com/hjson/hjson-py +Author: Christian Zangl +Author-email: laktak@cdak.net +License: MIT License +Keywords: json comments configuration +Platform: any +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: License :: OSI Approved :: Academic Free License (AFL) +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries :: Python Modules +License-File: LICENSE.txt + +hjson-py +======== + +`Hjson`_, a user interface for JSON + +Hjson works with Python 2.5+ and Python 3.3+ (based on `simplejson`_) + +Installation +============ + +- ``pip install hjson`` + +- or download from https://pypi.python.org/pypi/hjson + +Commandline +----------- + +:: + + Usage: + hjson [options] + hjson [options] + hjson (-h | --help) + hjson (-V | --version) + + Options: + -h --help Show this screen. + -j Output as formatted JSON. + -c Output as JSON. + -V --version Show version. + +E.g. ``echo '{"json":"obj"}' | hjson`` + + +Usage +===== + +.. code-block:: python + + import hjson + +Decoding Hjson +-------------- + +.. code-block:: python + + text = """{ + foo: a + bar: 1 + }""" + + hjson.loads(text) + +Result: + +.. code-block:: python + + OrderedDict([('foo', 'a'), ('bar', 1)]) + +Encoding Python object hierarchies +---------------------------------- + +.. code-block:: python + + hjson.dumps({'foo': 'text', 'bar': (1, 2)}) + +Result: + +:: + + { + foo: text + bar: + [ + 1 + 2 + ] + } + +Encoding as JSON +---------------- + +Note that this is probably not as performant as the simplejson version. + +.. code-block:: python + + hjson.dumpsJSON(['foo', {'bar': ('baz', None, 1.0, 2)}]) + +Result: ``'["foo", {"bar": ["baz", null, 1.0, 2]}]'`` + +API +=== + +`hjson-py`_ + +.. _Hjson: https://hjson.github.io +.. _simplejson: https://github.com/simplejson/simplejson +.. _hjson-py: http://hjson.github.io/hjson-py/ diff --git a/lib/python3.1/site-packages/hjson-3.1.0.dist-info/RECORD b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..84dcb36152f35bed65a01ceb66b4c64e8dbbe90e --- /dev/null +++ b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/RECORD @@ -0,0 +1,76 @@ +../../../bin/hjson,sha256=SS9FrgxHkjpNfdtSJS8qXAoJxVe6QYYSq868alLFpSo,243 +hjson-3.1.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +hjson-3.1.0.dist-info/LICENSE.txt,sha256=WRxR9JxoM7zhpdU-Y3haBGet1nRaWogOTRD05kzETEw,10462 +hjson-3.1.0.dist-info/METADATA,sha256=ws3nEqUXFpWNgmH-FYScGumUSgt2dkiPHNm7RZh-gcU,2610 +hjson-3.1.0.dist-info/RECORD,, +hjson-3.1.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92 +hjson-3.1.0.dist-info/entry_points.txt,sha256=sghBTcTrqdK0kKuznOZXeEBtjoBnLweHF83S9NK-r0E,42 +hjson-3.1.0.dist-info/top_level.txt,sha256=earxK7niVoW6ADlKRFHCs3JpUL4FBSnJn9qmrYgnEUE,6 +hjson/__init__.py,sha256=Py1drAVt-Xpi2xJAHxVXDdX-NniC4llxZVPjY7MyPCY,26580 +hjson/__pycache__/__init__.cpython-310.pyc,, +hjson/__pycache__/compat.cpython-310.pyc,, +hjson/__pycache__/decoder.cpython-310.pyc,, +hjson/__pycache__/encoder.cpython-310.pyc,, +hjson/__pycache__/encoderH.cpython-310.pyc,, +hjson/__pycache__/ordered_dict.cpython-310.pyc,, +hjson/__pycache__/scanner.cpython-310.pyc,, +hjson/__pycache__/tool.cpython-310.pyc,, +hjson/compat.py,sha256=uvmTYe1Oa18tT_6tFRtYnzTdAkyd73B9zwMF7ZafI04,1036 +hjson/decoder.py,sha256=oHz7g2sQd4S-AJbSzur9VJWqAHDWL25FVQ-G35XlGCA,19563 +hjson/encoder.py,sha256=NhR3YSMVzL3UP8OLAtP2Dr6dW9UbJzjVS-SXp1DzvcY,19168 +hjson/encoderH.py,sha256=wJ8D0gAyR3n6e3MahCIU3OahI5_xuHPWV_ZlI839xCs,20481 +hjson/ordered_dict.py,sha256=DXtgiqkkaNWXDLZ0DGXIjF_CPzGV5qpC-PSeS1zcps8,3370 +hjson/scanner.py,sha256=IL8poQGvCsb82y7qY5jrlSrZ5xcENpPUti3tNKhprYw,1779 +hjson/tests/__init__.py,sha256=_A-1Tn7q7ccNPro_QfbKiXo_bTL9ED5RUX9AeSLG4TA,2011 +hjson/tests/__pycache__/__init__.cpython-310.pyc,, +hjson/tests/__pycache__/test_bigint_as_string.cpython-310.pyc,, +hjson/tests/__pycache__/test_bitsize_int_as_string.cpython-310.pyc,, +hjson/tests/__pycache__/test_check_circular.cpython-310.pyc,, +hjson/tests/__pycache__/test_decimal.cpython-310.pyc,, +hjson/tests/__pycache__/test_decode.cpython-310.pyc,, +hjson/tests/__pycache__/test_default.cpython-310.pyc,, +hjson/tests/__pycache__/test_dump.cpython-310.pyc,, +hjson/tests/__pycache__/test_encode_basestring_ascii.cpython-310.pyc,, +hjson/tests/__pycache__/test_errors.cpython-310.pyc,, +hjson/tests/__pycache__/test_fail.cpython-310.pyc,, +hjson/tests/__pycache__/test_float.cpython-310.pyc,, +hjson/tests/__pycache__/test_for_json.cpython-310.pyc,, +hjson/tests/__pycache__/test_hjson.cpython-310.pyc,, +hjson/tests/__pycache__/test_indent.cpython-310.pyc,, +hjson/tests/__pycache__/test_item_sort_key.cpython-310.pyc,, +hjson/tests/__pycache__/test_namedtuple.cpython-310.pyc,, +hjson/tests/__pycache__/test_pass1.cpython-310.pyc,, +hjson/tests/__pycache__/test_pass2.cpython-310.pyc,, +hjson/tests/__pycache__/test_pass3.cpython-310.pyc,, +hjson/tests/__pycache__/test_recursion.cpython-310.pyc,, +hjson/tests/__pycache__/test_scanstring.cpython-310.pyc,, +hjson/tests/__pycache__/test_separators.cpython-310.pyc,, +hjson/tests/__pycache__/test_tool.cpython-310.pyc,, +hjson/tests/__pycache__/test_tuple.cpython-310.pyc,, +hjson/tests/__pycache__/test_unicode.cpython-310.pyc,, +hjson/tests/test_bigint_as_string.py,sha256=bhEtYEXWUhxyi25iLm4sPWFrt5RZ9PfFEknX1cdzP-Y,2265 +hjson/tests/test_bitsize_int_as_string.py,sha256=-73xJ8CE2hDjGOWTERRrYbDHaY0kgBNQXC0g8nIgd4k,2332 +hjson/tests/test_check_circular.py,sha256=64kZhsab6OcwYmJNLRqNW-19dp1UdgYbZiGzopKyR9s,940 +hjson/tests/test_decimal.py,sha256=Qw0IBPSPYoGZXwvXkkM1cz6lpqjkPRzNDBSul-RdR_4,2556 +hjson/tests/test_decode.py,sha256=Sm4052xVjv7ZtZFdRVMsnvQeh2eCNoXv24YOUJJLMdg,4437 +hjson/tests/test_default.py,sha256=WWDLhDVfih4PrenmiEcvshhUOl_bNsm3jML96-AtGmo,224 +hjson/tests/test_dump.py,sha256=5WU4Rd6vsHOwXGpGqQKIw1ZBNgRWUqMY8w3DnJVWfxo,5061 +hjson/tests/test_encode_basestring_ascii.py,sha256=up4y9JMdGXdBXkEjfqwiG-sudSdcKw0RQfO_76za-To,2102 +hjson/tests/test_errors.py,sha256=vg3-z36T9O-UeDHG4ZtW-nQBNAvraWKBrDA70yG989c,1549 +hjson/tests/test_fail.py,sha256=Giinb944NX0bPwBHYUjVZ4ZlNB611Wg0wxVWxv4bDaU,5688 +hjson/tests/test_float.py,sha256=LCUL-2xT8PYq99jQi6-Ddk9pMuC1mLrcJboTfvR08HM,1011 +hjson/tests/test_for_json.py,sha256=ZLtypdX0ALctxMB8c3fQvx3k9OHY5t71gBxGNOXemrc,2778 +hjson/tests/test_hjson.py,sha256=CdvXR05nu8bF_jZ-Hhj3bh8LRi8tdSJTruayj69HoDk,2327 +hjson/tests/test_indent.py,sha256=8oUK5E8DTz1c3RkUU-nOELmr9wOKoaHHOAsxDai66iE,2589 +hjson/tests/test_item_sort_key.py,sha256=piYy-ntwdcb_qS-y8jPFI6rVZlHCNqtTFGnaZSEvWH8,1134 +hjson/tests/test_namedtuple.py,sha256=iK7B95JH4f2L3_MB3rY9NagEVZ1X62JHpjlm1J4t5uM,4066 +hjson/tests/test_pass1.py,sha256=wdnBz55dY4ou8IIdZFypJ_72J6HCtLZw0YesoNOTopQ,1745 +hjson/tests/test_pass2.py,sha256=O389C8IeJ9ysqWKUftXOeHvBi5_47VSdBM_2bCee1SQ,385 +hjson/tests/test_pass3.py,sha256=wwemRholrfv4lwgC0ArnXiS_tfA-NnbRrZqo71YZXaA,481 +hjson/tests/test_recursion.py,sha256=APQAtnchO6KZdduYqvEJDrBOmAKqyORwAX4ldbzAW7A,1694 +hjson/tests/test_scanstring.py,sha256=U-5gUsaO7jEai6zfKMKADfBi-xFg8IHC1dBeElvTsn4,5953 +hjson/tests/test_separators.py,sha256=krudtyZbx0k7OrZZ-jGCZH6yrmsj1seRkUblQLRq4Rw,945 +hjson/tests/test_tool.py,sha256=PyDhrmMTpJSvBXvsWG8P-yrw1BT5kSp6OcSrMLvzmIw,2826 +hjson/tests/test_tuple.py,sha256=FkOyw4s7WZ1HUGxtHYiwaBwhW2thlDB3JFSpUZCbI_I,1976 +hjson/tests/test_unicode.py,sha256=5npz25rj4T3ZhXF_OsSmVveBUtTuz92DQ_cHDe-Jfrw,7099 +hjson/tool.py,sha256=PVlr-lSDWfrGdkh5exWQo_21HGgHLR4OpZ1S41Dqjgo,1897 diff --git a/lib/python3.1/site-packages/hjson-3.1.0.dist-info/WHEEL b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..becc9a66ea739ba941d48a749e248761cc6e658a --- /dev/null +++ b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.37.1) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/lib/python3.1/site-packages/hjson-3.1.0.dist-info/entry_points.txt b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..f8b7adb03e6805f68c21d7b601e0ac6f8e823e38 --- /dev/null +++ b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +hjson = hjson.tool:main diff --git a/lib/python3.1/site-packages/hjson-3.1.0.dist-info/top_level.txt b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..4b1c5746ac3bdad0432afd48512a61f319fd74cc --- /dev/null +++ b/lib/python3.1/site-packages/hjson-3.1.0.dist-info/top_level.txt @@ -0,0 +1 @@ +hjson diff --git a/lib/python3.1/site-packages/interegular-0.3.3.dist-info/INSTALLER b/lib/python3.1/site-packages/interegular-0.3.3.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.1/site-packages/interegular-0.3.3.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.1/site-packages/interegular-0.3.3.dist-info/LICENSE.txt b/lib/python3.1/site-packages/interegular-0.3.3.dist-info/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..daacff994e85109a1756852fb20695e12201ebc6 --- /dev/null +++ b/lib/python3.1/site-packages/interegular-0.3.3.dist-info/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 @MegaIng and @qntm + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lib/python3.1/site-packages/interegular-0.3.3.dist-info/METADATA b/lib/python3.1/site-packages/interegular-0.3.3.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..d61c48a2ebadce5bc0a627238171131864e7524d --- /dev/null +++ b/lib/python3.1/site-packages/interegular-0.3.3.dist-info/METADATA @@ -0,0 +1,64 @@ +Metadata-Version: 2.1 +Name: interegular +Version: 0.3.3 +Summary: a regex intersection checker +Home-page: https://github.com/MegaIng/regex_intersections +Download-URL: https://github.com/MegaIng/interegular/tarball/master +Author: MegaIng +Author-email: MegaIng +License: MIT +Classifier: Operating System :: OS Independent +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Requires-Python: >=3.7 +Description-Content-Type: text/markdown +License-File: LICENSE.txt + +# Interegular +***regex intersection checker*** + +A library to check a subset of python regexes for intersections. +Based on [grennery](https://github.com/qntm/greenery) by [@qntm](https://github.com/qntm). Adapted for [lark-parser](https://github.com/lark-parser/lark). + +The primary difference with `grennery` library is that `interegular` is focused on speed and compatibility with python re syntax, whereas grennery has a way to reconstruct a regex from a FSM, which `interegular` lacks. + + +## Interface + +| Function | Usage | +| -------- | ----- | +| `compare_regexes(*regexes: str)` | Takes a series of regexes as strings and returns a Generator of all intersections as `(str, str)`| +| `parse_pattern(regex: str)` | Parses a regex as string to a `Pattern` object| +| `interegular.compare_patterns(*patterns: Pattern)` | Takes a series of regexes as patterns and returns a Generator of all intersections as `(Pattern, Pattern)`| +| `Pattern` | A class representing a parsed regex (intermediate representation)| +| `REFlags` | A enum representing the flags a regex can have | +| `FSM` | A class representing a fully parsed regex. (Has many useful members) | +| `Pattern.with_flags(added: REFlags, removed: REFlags)` | A function to change the flags that are applied to a regex| +| `Pattern.to_fsm() -> FSM` | A function to create a `FSM` object from the Pattern | +| `Comparator` | A Class to compare a group of Patterns | + +## What is supported? + +Most normal python-regex syntax is support. But because of the backend that is used (final-state-machines), some things can not be implemented. This includes: + +- Backwards references (`\1`, `(?P=open)`) +- Conditional Matching (`(?(1)a|b)`) +- Some cases of lookaheads/lookbacks (You gotta try out which work and which don't) + - A word of warning: This is currently not correctly handled, and some things might parse, but not work correctly. I am currently working on this. + + +Some things are simply not implemented and will implemented in the future: +- Some flags (Progress: `ims` from `aiLmsux`) +- Some cases of lookaheads/lookbacks (You gotta try out which work and which don't) + + +## TODO + +- Docs +- More tests +- Checks that the syntax is correctly handled. diff --git a/lib/python3.1/site-packages/interegular-0.3.3.dist-info/RECORD b/lib/python3.1/site-packages/interegular-0.3.3.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..bf937b5a0a217337330ee12cebac89682e8a7917 --- /dev/null +++ b/lib/python3.1/site-packages/interegular-0.3.3.dist-info/RECORD @@ -0,0 +1,19 @@ +interegular-0.3.3.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +interegular-0.3.3.dist-info/LICENSE.txt,sha256=iiMYmhB0ePopJBii4LG4vS_mBhfBzmMlwhlPbZHyeyk,1096 +interegular-0.3.3.dist-info/METADATA,sha256=x1mGaPlws3xtIWEswqvJJTFWv4tcOQVSR5PtmvMJrLM,3049 +interegular-0.3.3.dist-info/RECORD,, +interegular-0.3.3.dist-info/WHEEL,sha256=9FUYUSCt5io1tVBOVa-3S0zjcMzGO1nOIw99iWozm3o,93 +interegular-0.3.3.dist-info/top_level.txt,sha256=A98LQ5TOfM4ZxEotuKyytmx17Y3u_ujlxjsOZTa-AhQ,12 +interegular/__init__.py,sha256=TmnSMLjCnSxwJYL9yNW-65qaZuhwuCjfrR20eXrq3pQ,1128 +interegular/__pycache__/__init__.cpython-310.pyc,, +interegular/__pycache__/comparator.cpython-310.pyc,, +interegular/__pycache__/fsm.cpython-310.pyc,, +interegular/__pycache__/patterns.cpython-310.pyc,, +interegular/comparator.py,sha256=tXm7f6n4GPbZrl69svAR_n-Jewwwe1JnjZNm1tr1HHw,7198 +interegular/fsm.py,sha256=CD-cZq6pB3fpCcczbG5-_4sUbIyd-a62xmMKysgBQaw,39130 +interegular/patterns.py,sha256=vf1-bkjNicV9ktlFujBg95rPp95DAypFDeWqPeamXRY,25902 +interegular/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +interegular/utils/__init__.py,sha256=co6xcnq7CgWZ6SLP305ToStLskyr3R3NxYUfiOq37eQ,451 +interegular/utils/__pycache__/__init__.cpython-310.pyc,, +interegular/utils/__pycache__/simple_parser.cpython-310.pyc,, +interegular/utils/simple_parser.py,sha256=G-CRsKKmekN5I9FI8g37bl1av9aESFoyJTmKCOIHuzQ,5361 diff --git a/lib/python3.1/site-packages/interegular-0.3.3.dist-info/WHEEL b/lib/python3.1/site-packages/interegular-0.3.3.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..90ea5671dccc98ec1c4f872cea2efc6608a7a519 --- /dev/null +++ b/lib/python3.1/site-packages/interegular-0.3.3.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.42.0) +Root-Is-Purelib: true +Tag: py37-none-any + diff --git a/lib/python3.1/site-packages/interegular-0.3.3.dist-info/top_level.txt b/lib/python3.1/site-packages/interegular-0.3.3.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..762bfabc9d13020947cc4761546e4cac09af0df2 --- /dev/null +++ b/lib/python3.1/site-packages/interegular-0.3.3.dist-info/top_level.txt @@ -0,0 +1 @@ +interegular diff --git a/lib/python3.1/site-packages/isympy.py b/lib/python3.1/site-packages/isympy.py new file mode 100644 index 0000000000000000000000000000000000000000..50e9bc78d08904b8c177105ee90d984ea4b01d20 --- /dev/null +++ b/lib/python3.1/site-packages/isympy.py @@ -0,0 +1,342 @@ +""" +Python shell for SymPy. + +This is just a normal Python shell (IPython shell if you have the +IPython package installed), that executes the following commands for +the user: + + >>> from __future__ import division + >>> from sympy import * + >>> x, y, z, t = symbols('x y z t') + >>> k, m, n = symbols('k m n', integer=True) + >>> f, g, h = symbols('f g h', cls=Function) + >>> init_printing() + +So starting 'isympy' is equivalent to starting Python (or IPython) and +executing the above commands by hand. It is intended for easy and quick +experimentation with SymPy. isympy is a good way to use SymPy as an +interactive calculator. If you have IPython and Matplotlib installed, then +interactive plotting is enabled by default. + +COMMAND LINE OPTIONS +-------------------- + +-c CONSOLE, --console=CONSOLE + + Use the specified shell (Python or IPython) shell as the console + backend instead of the default one (IPython if present, Python + otherwise), e.g.: + + $isympy -c python + + CONSOLE must be one of 'ipython' or 'python' + +-p PRETTY, --pretty PRETTY + + Setup pretty-printing in SymPy. When pretty-printing is enabled, + expressions can be printed with Unicode or ASCII. The default is + to use pretty-printing (with Unicode if the terminal supports it). + When this option is 'no', expressions will not be pretty-printed + and ASCII will be used: + + $isympy -p no + + PRETTY must be one of 'unicode', 'ascii', or 'no' + +-t TYPES, --types=TYPES + + Setup the ground types for the polys. By default, gmpy ground types + are used if gmpy2 or gmpy is installed, otherwise it falls back to python + ground types, which are a little bit slower. You can manually + choose python ground types even if gmpy is installed (e.g., for + testing purposes): + + $isympy -t python + + TYPES must be one of 'gmpy', 'gmpy1' or 'python' + + Note that the ground type gmpy1 is primarily intended for testing; it + forces the use of gmpy version 1 even if gmpy2 is available. + + This is the same as setting the environment variable + SYMPY_GROUND_TYPES to the given ground type (e.g., + SYMPY_GROUND_TYPES='gmpy') + + The ground types can be determined interactively from the variable + sympy.polys.domains.GROUND_TYPES. + +-o ORDER, --order ORDER + + Setup the ordering of terms for printing. The default is lex, which + orders terms lexicographically (e.g., x**2 + x + 1). You can choose + other orderings, such as rev-lex, which will use reverse + lexicographic ordering (e.g., 1 + x + x**2): + + $isympy -o rev-lex + + ORDER must be one of 'lex', 'rev-lex', 'grlex', 'rev-grlex', + 'grevlex', 'rev-grevlex', 'old', or 'none'. + + Note that for very large expressions, ORDER='none' may speed up + printing considerably but the terms will have no canonical order. + +-q, --quiet + + Print only Python's and SymPy's versions to stdout at startup. + +-d, --doctest + + Use the same format that should be used for doctests. This is + equivalent to -c python -p no. + +-C, --no-cache + + Disable the caching mechanism. Disabling the cache may slow certain + operations down considerably. This is useful for testing the cache, + or for benchmarking, as the cache can result in deceptive timings. + + This is equivalent to setting the environment variable + SYMPY_USE_CACHE to 'no'. + +-a, --auto-symbols (requires at least IPython 0.11) + + Automatically create missing symbols. Normally, typing a name of a + Symbol that has not been instantiated first would raise NameError, + but with this option enabled, any undefined name will be + automatically created as a Symbol. + + Note that this is intended only for interactive, calculator style + usage. In a script that uses SymPy, Symbols should be instantiated + at the top, so that it's clear what they are. + + This will not override any names that are already defined, which + includes the single character letters represented by the mnemonic + QCOSINE (see the "Gotchas and Pitfalls" document in the + documentation). You can delete existing names by executing "del + name". If a name is defined, typing "'name' in dir()" will return True. + + The Symbols that are created using this have default assumptions. + If you want to place assumptions on symbols, you should create them + using symbols() or var(). + + Finally, this only works in the top level namespace. So, for + example, if you define a function in isympy with an undefined + Symbol, it will not work. + + See also the -i and -I options. + +-i, --int-to-Integer (requires at least IPython 0.11) + + Automatically wrap int literals with Integer. This makes it so that + things like 1/2 will come out as Rational(1, 2), rather than 0.5. This + works by preprocessing the source and wrapping all int literals with + Integer. Note that this will not change the behavior of int literals + assigned to variables, and it also won't change the behavior of functions + that return int literals. + + If you want an int, you can wrap the literal in int(), e.g. int(3)/int(2) + gives 1.5 (with division imported from __future__). + +-I, --interactive (requires at least IPython 0.11) + + This is equivalent to --auto-symbols --int-to-Integer. Future options + designed for ease of interactive use may be added to this. + +-D, --debug + + Enable debugging output. This is the same as setting the + environment variable SYMPY_DEBUG to 'True'. The debug status is set + in the variable SYMPY_DEBUG within isympy. + +-- IPython options + + Additionally you can pass command line options directly to the IPython + interpreter (the standard Python shell is not supported). However you + need to add the '--' separator between two types of options, e.g the + startup banner option and the colors option. You need to enter the + options as required by the version of IPython that you are using, too: + + in IPython 0.11, + + $isympy -q -- --colors=NoColor + + or older versions of IPython, + + $isympy -q -- -colors NoColor + +See also isympy --help. +""" + +import os +import sys + +# DO NOT IMPORT SYMPY HERE! Or the setting of the sympy environment variables +# by the command line will break. + +def main() -> None: + from argparse import ArgumentParser, RawDescriptionHelpFormatter + + VERSION = None + if '--version' in sys.argv: + # We cannot import sympy before this is run, because flags like -C and + # -t set environment variables that must be set before SymPy is + # imported. The only thing we need to import it for is to get the + # version, which only matters with the --version flag. + import sympy + VERSION = sympy.__version__ + + usage = 'isympy [options] -- [ipython options]' + parser = ArgumentParser( + usage=usage, + description=__doc__, + formatter_class=RawDescriptionHelpFormatter, + ) + + parser.add_argument('--version', action='version', version=VERSION) + + parser.add_argument( + '-c', '--console', + dest='console', + action='store', + default=None, + choices=['ipython', 'python'], + metavar='CONSOLE', + help='select type of interactive session: ipython | python; defaults ' + 'to ipython if IPython is installed, otherwise python') + + parser.add_argument( + '-p', '--pretty', + dest='pretty', + action='store', + default=None, + metavar='PRETTY', + choices=['unicode', 'ascii', 'no'], + help='setup pretty printing: unicode | ascii | no; defaults to ' + 'unicode printing if the terminal supports it, otherwise ascii') + + parser.add_argument( + '-t', '--types', + dest='types', + action='store', + default=None, + metavar='TYPES', + choices=['gmpy', 'gmpy1', 'python'], + help='setup ground types: gmpy | gmpy1 | python; defaults to gmpy if gmpy2 ' + 'or gmpy is installed, otherwise python') + + parser.add_argument( + '-o', '--order', + dest='order', + action='store', + default=None, + metavar='ORDER', + choices=['lex', 'grlex', 'grevlex', 'rev-lex', 'rev-grlex', 'rev-grevlex', 'old', 'none'], + help='setup ordering of terms: [rev-]lex | [rev-]grlex | [rev-]grevlex | old | none; defaults to lex') + + parser.add_argument( + '-q', '--quiet', + dest='quiet', + action='store_true', + default=False, + help='print only version information at startup') + + parser.add_argument( + '-d', '--doctest', + dest='doctest', + action='store_true', + default=False, + help='use the doctest format for output (you can just copy and paste it)') + + parser.add_argument( + '-C', '--no-cache', + dest='cache', + action='store_false', + default=True, + help='disable caching mechanism') + + parser.add_argument( + '-a', '--auto-symbols', + dest='auto_symbols', + action='store_true', + default=False, + help='automatically construct missing symbols') + + parser.add_argument( + '-i', '--int-to-Integer', + dest='auto_int_to_Integer', + action='store_true', + default=False, + help="automatically wrap int literals with Integer") + + parser.add_argument( + '-I', '--interactive', + dest='interactive', + action='store_true', + default=False, + help="equivalent to -a -i") + + parser.add_argument( + '-D', '--debug', + dest='debug', + action='store_true', + default=False, + help='enable debugging output') + + (options, ipy_args) = parser.parse_known_args() + if '--' in ipy_args: + ipy_args.remove('--') + + if not options.cache: + os.environ['SYMPY_USE_CACHE'] = 'no' + + if options.types: + os.environ['SYMPY_GROUND_TYPES'] = options.types + + if options.debug: + os.environ['SYMPY_DEBUG'] = str(options.debug) + + if options.doctest: + options.pretty = 'no' + options.console = 'python' + + session = options.console + + if session is not None: + ipython = session == 'ipython' + else: + try: + import IPython + ipython = True + except ImportError: + if not options.quiet: + from sympy.interactive.session import no_ipython + print(no_ipython) + ipython = False + + args = { + 'pretty_print': True, + 'use_unicode': None, + 'use_latex': None, + 'order': None, + 'argv': ipy_args, + } + + if options.pretty == 'unicode': + args['use_unicode'] = True + elif options.pretty == 'ascii': + args['use_unicode'] = False + elif options.pretty == 'no': + args['pretty_print'] = False + + if options.order is not None: + args['order'] = options.order + + args['quiet'] = options.quiet + args['auto_symbols'] = options.auto_symbols or options.interactive + args['auto_int_to_Integer'] = options.auto_int_to_Integer or options.interactive + + from sympy.interactive import init_session + init_session(ipython, **args) + +if __name__ == "__main__": + main() diff --git a/lib/python3.1/site-packages/mdurl/__init__.py b/lib/python3.1/site-packages/mdurl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cdbb640e004cef0e950a656a53d92d89d82c7472 --- /dev/null +++ b/lib/python3.1/site-packages/mdurl/__init__.py @@ -0,0 +1,18 @@ +__all__ = ( + "decode", + "DECODE_DEFAULT_CHARS", + "DECODE_COMPONENT_CHARS", + "encode", + "ENCODE_DEFAULT_CHARS", + "ENCODE_COMPONENT_CHARS", + "format", + "parse", + "URL", +) +__version__ = "0.1.2" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT + +from mdurl._decode import DECODE_COMPONENT_CHARS, DECODE_DEFAULT_CHARS, decode +from mdurl._encode import ENCODE_COMPONENT_CHARS, ENCODE_DEFAULT_CHARS, encode +from mdurl._format import format +from mdurl._parse import url_parse as parse +from mdurl._url import URL diff --git a/lib/python3.1/site-packages/mdurl/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/site-packages/mdurl/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..068ff29ecea3ea7663b6fe1af58dd36769dfd5cf Binary files /dev/null and b/lib/python3.1/site-packages/mdurl/__pycache__/__init__.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/mdurl/__pycache__/_decode.cpython-310.pyc b/lib/python3.1/site-packages/mdurl/__pycache__/_decode.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..728e45356fabd4afab4e4f4eb4883332d31b763b Binary files /dev/null and b/lib/python3.1/site-packages/mdurl/__pycache__/_decode.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/mdurl/__pycache__/_encode.cpython-310.pyc b/lib/python3.1/site-packages/mdurl/__pycache__/_encode.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..78e6c24061b4de152a781aeacf7d74505dc013b7 Binary files /dev/null and b/lib/python3.1/site-packages/mdurl/__pycache__/_encode.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/mdurl/__pycache__/_format.cpython-310.pyc b/lib/python3.1/site-packages/mdurl/__pycache__/_format.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4b044ffadad53815b26cb9c918a12a16eeba74de Binary files /dev/null and b/lib/python3.1/site-packages/mdurl/__pycache__/_format.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/mdurl/__pycache__/_parse.cpython-310.pyc b/lib/python3.1/site-packages/mdurl/__pycache__/_parse.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61ef666e2f2779ce07f13c2883d28bff3179225f Binary files /dev/null and b/lib/python3.1/site-packages/mdurl/__pycache__/_parse.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/mdurl/__pycache__/_url.cpython-310.pyc b/lib/python3.1/site-packages/mdurl/__pycache__/_url.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f6c6de442a2cbc5047cd81fadcb233dc19aa147 Binary files /dev/null and b/lib/python3.1/site-packages/mdurl/__pycache__/_url.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/mdurl/_decode.py b/lib/python3.1/site-packages/mdurl/_decode.py new file mode 100644 index 0000000000000000000000000000000000000000..9b50a2dde976a6d43491ec6f20d12e60f6f6597f --- /dev/null +++ b/lib/python3.1/site-packages/mdurl/_decode.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from collections.abc import Sequence +import functools +import re + +DECODE_DEFAULT_CHARS = ";/?:@&=+$,#" +DECODE_COMPONENT_CHARS = "" + +decode_cache: dict[str, list[str]] = {} + + +def get_decode_cache(exclude: str) -> Sequence[str]: + if exclude in decode_cache: + return decode_cache[exclude] + + cache: list[str] = [] + decode_cache[exclude] = cache + + for i in range(128): + ch = chr(i) + cache.append(ch) + + for i in range(len(exclude)): + ch_code = ord(exclude[i]) + cache[ch_code] = "%" + ("0" + hex(ch_code)[2:].upper())[-2:] + + return cache + + +# Decode percent-encoded string. +# +def decode(string: str, exclude: str = DECODE_DEFAULT_CHARS) -> str: + cache = get_decode_cache(exclude) + repl_func = functools.partial(repl_func_with_cache, cache=cache) + return re.sub(r"(%[a-f0-9]{2})+", repl_func, string, flags=re.IGNORECASE) + + +def repl_func_with_cache(match: re.Match, cache: Sequence[str]) -> str: + seq = match.group() + result = "" + + i = 0 + l = len(seq) # noqa: E741 + while i < l: + b1 = int(seq[i + 1 : i + 3], 16) + + if b1 < 0x80: + result += cache[b1] + i += 3 # emulate JS for loop statement3 + continue + + if (b1 & 0xE0) == 0xC0 and (i + 3 < l): + # 110xxxxx 10xxxxxx + b2 = int(seq[i + 4 : i + 6], 16) + + if (b2 & 0xC0) == 0x80: + all_bytes = bytes((b1, b2)) + try: + result += all_bytes.decode() + except UnicodeDecodeError: + result += "\ufffd" * 2 + + i += 3 + i += 3 # emulate JS for loop statement3 + continue + + if (b1 & 0xF0) == 0xE0 and (i + 6 < l): + # 1110xxxx 10xxxxxx 10xxxxxx + b2 = int(seq[i + 4 : i + 6], 16) + b3 = int(seq[i + 7 : i + 9], 16) + + if (b2 & 0xC0) == 0x80 and (b3 & 0xC0) == 0x80: + all_bytes = bytes((b1, b2, b3)) + try: + result += all_bytes.decode() + except UnicodeDecodeError: + result += "\ufffd" * 3 + + i += 6 + i += 3 # emulate JS for loop statement3 + continue + + if (b1 & 0xF8) == 0xF0 and (i + 9 < l): + # 111110xx 10xxxxxx 10xxxxxx 10xxxxxx + b2 = int(seq[i + 4 : i + 6], 16) + b3 = int(seq[i + 7 : i + 9], 16) + b4 = int(seq[i + 10 : i + 12], 16) + + if (b2 & 0xC0) == 0x80 and (b3 & 0xC0) == 0x80 and (b4 & 0xC0) == 0x80: + all_bytes = bytes((b1, b2, b3, b4)) + try: + result += all_bytes.decode() + except UnicodeDecodeError: + result += "\ufffd" * 4 + + i += 9 + i += 3 # emulate JS for loop statement3 + continue + + result += "\ufffd" + i += 3 # emulate JS for loop statement3 + + return result diff --git a/lib/python3.1/site-packages/mdurl/_encode.py b/lib/python3.1/site-packages/mdurl/_encode.py new file mode 100644 index 0000000000000000000000000000000000000000..bc2e5b917afe9e9ecaa6f11af7a9ac82704d3914 --- /dev/null +++ b/lib/python3.1/site-packages/mdurl/_encode.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from collections.abc import Sequence +from string import ascii_letters, digits, hexdigits +from urllib.parse import quote as encode_uri_component + +ASCII_LETTERS_AND_DIGITS = ascii_letters + digits + +ENCODE_DEFAULT_CHARS = ";/?:@&=+$,-_.!~*'()#" +ENCODE_COMPONENT_CHARS = "-_.!~*'()" + +encode_cache: dict[str, list[str]] = {} + + +# Create a lookup array where anything but characters in `chars` string +# and alphanumeric chars is percent-encoded. +def get_encode_cache(exclude: str) -> Sequence[str]: + if exclude in encode_cache: + return encode_cache[exclude] + + cache: list[str] = [] + encode_cache[exclude] = cache + + for i in range(128): + ch = chr(i) + + if ch in ASCII_LETTERS_AND_DIGITS: + # always allow unencoded alphanumeric characters + cache.append(ch) + else: + cache.append("%" + ("0" + hex(i)[2:].upper())[-2:]) + + for i in range(len(exclude)): + cache[ord(exclude[i])] = exclude[i] + + return cache + + +# Encode unsafe characters with percent-encoding, skipping already +# encoded sequences. +# +# - string - string to encode +# - exclude - list of characters to ignore (in addition to a-zA-Z0-9) +# - keepEscaped - don't encode '%' in a correct escape sequence (default: true) +def encode( + string: str, exclude: str = ENCODE_DEFAULT_CHARS, *, keep_escaped: bool = True +) -> str: + result = "" + + cache = get_encode_cache(exclude) + + l = len(string) # noqa: E741 + i = 0 + while i < l: + code = ord(string[i]) + + # % + if keep_escaped and code == 0x25 and i + 2 < l: + if all(c in hexdigits for c in string[i + 1 : i + 3]): + result += string[i : i + 3] + i += 2 + i += 1 # JS for loop statement3 + continue + + if code < 128: + result += cache[code] + i += 1 # JS for loop statement3 + continue + + if code >= 0xD800 and code <= 0xDFFF: + if code >= 0xD800 and code <= 0xDBFF and i + 1 < l: + next_code = ord(string[i + 1]) + if next_code >= 0xDC00 and next_code <= 0xDFFF: + result += encode_uri_component(string[i] + string[i + 1]) + i += 1 + i += 1 # JS for loop statement3 + continue + result += "%EF%BF%BD" + i += 1 # JS for loop statement3 + continue + + result += encode_uri_component(string[i]) + i += 1 # JS for loop statement3 + + return result diff --git a/lib/python3.1/site-packages/mdurl/_format.py b/lib/python3.1/site-packages/mdurl/_format.py new file mode 100644 index 0000000000000000000000000000000000000000..12524ca626065183ec9974f3d7d08dadd4a7d3e8 --- /dev/null +++ b/lib/python3.1/site-packages/mdurl/_format.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from mdurl._url import URL + + +def format(url: URL) -> str: # noqa: A001 + result = "" + + result += url.protocol or "" + result += "//" if url.slashes else "" + result += url.auth + "@" if url.auth else "" + + if url.hostname and ":" in url.hostname: + # ipv6 address + result += "[" + url.hostname + "]" + else: + result += url.hostname or "" + + result += ":" + url.port if url.port else "" + result += url.pathname or "" + result += url.search or "" + result += url.hash or "" + + return result diff --git a/lib/python3.1/site-packages/mdurl/_parse.py b/lib/python3.1/site-packages/mdurl/_parse.py new file mode 100644 index 0000000000000000000000000000000000000000..ffeeac768dca3bff60c55c9b1f0bc0fbb4cec7b1 --- /dev/null +++ b/lib/python3.1/site-packages/mdurl/_parse.py @@ -0,0 +1,304 @@ +# Copyright Joyent, Inc. and other Node contributors. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the +# following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +# USE OR OTHER DEALINGS IN THE SOFTWARE. + + +# Changes from joyent/node: +# +# 1. No leading slash in paths, +# e.g. in `url.parse('http://foo?bar')` pathname is ``, not `/` +# +# 2. Backslashes are not replaced with slashes, +# so `http:\\example.org\` is treated like a relative path +# +# 3. Trailing colon is treated like a part of the path, +# i.e. in `http://example.org:foo` pathname is `:foo` +# +# 4. Nothing is URL-encoded in the resulting object, +# (in joyent/node some chars in auth and paths are encoded) +# +# 5. `url.parse()` does not have `parseQueryString` argument +# +# 6. Removed extraneous result properties: `host`, `path`, `query`, etc., +# which can be constructed using other parts of the url. + +from __future__ import annotations + +from collections import defaultdict +import re + +from mdurl._url import URL + +# Reference: RFC 3986, RFC 1808, RFC 2396 + +# define these here so at least they only have to be +# compiled once on the first module load. +PROTOCOL_PATTERN = re.compile(r"^([a-z0-9.+-]+:)", flags=re.IGNORECASE) +PORT_PATTERN = re.compile(r":[0-9]*$") + +# Special case for a simple path URL +SIMPLE_PATH_PATTERN = re.compile(r"^(//?(?!/)[^?\s]*)(\?[^\s]*)?$") + +# RFC 2396: characters reserved for delimiting URLs. +# We actually just auto-escape these. +DELIMS = ("<", ">", '"', "`", " ", "\r", "\n", "\t") + +# RFC 2396: characters not allowed for various reasons. +UNWISE = ("{", "}", "|", "\\", "^", "`") + DELIMS + +# Allowed by RFCs, but cause of XSS attacks. Always escape these. +AUTO_ESCAPE = ("'",) + UNWISE +# Characters that are never ever allowed in a hostname. +# Note that any invalid chars are also handled, but these +# are the ones that are *expected* to be seen, so we fast-path +# them. +NON_HOST_CHARS = ("%", "/", "?", ";", "#") + AUTO_ESCAPE +HOST_ENDING_CHARS = ("/", "?", "#") +HOSTNAME_MAX_LEN = 255 +HOSTNAME_PART_PATTERN = re.compile(r"^[+a-z0-9A-Z_-]{0,63}$") +HOSTNAME_PART_START = re.compile(r"^([+a-z0-9A-Z_-]{0,63})(.*)$") +# protocols that can allow "unsafe" and "unwise" chars. + +# protocols that never have a hostname. +HOSTLESS_PROTOCOL = defaultdict( + bool, + { + "javascript": True, + "javascript:": True, + }, +) +# protocols that always contain a // bit. +SLASHED_PROTOCOL = defaultdict( + bool, + { + "http": True, + "https": True, + "ftp": True, + "gopher": True, + "file": True, + "http:": True, + "https:": True, + "ftp:": True, + "gopher:": True, + "file:": True, + }, +) + + +class MutableURL: + def __init__(self) -> None: + self.protocol: str | None = None + self.slashes: bool = False + self.auth: str | None = None + self.port: str | None = None + self.hostname: str | None = None + self.hash: str | None = None + self.search: str | None = None + self.pathname: str | None = None + + def parse(self, url: str, slashes_denote_host: bool) -> "MutableURL": + lower_proto = "" + slashes = False + rest = url + + # trim before proceeding. + # This is to support parse stuff like " http://foo.com \n" + rest = rest.strip() + + if not slashes_denote_host and len(url.split("#")) == 1: + # Try fast path regexp + simple_path = SIMPLE_PATH_PATTERN.match(rest) + if simple_path: + self.pathname = simple_path.group(1) + if simple_path.group(2): + self.search = simple_path.group(2) + return self + + proto = "" + proto_match = PROTOCOL_PATTERN.match(rest) + if proto_match: + proto = proto_match.group() + lower_proto = proto.lower() + self.protocol = proto + rest = rest[len(proto) :] + + # figure out if it's got a host + # user@server is *always* interpreted as a hostname, and url + # resolution will treat //foo/bar as host=foo,path=bar because that's + # how the browser resolves relative URLs. + if slashes_denote_host or proto or re.search(r"^//[^@/]+@[^@/]+", rest): + slashes = rest.startswith("//") + if slashes and not (proto and HOSTLESS_PROTOCOL[proto]): + rest = rest[2:] + self.slashes = True + + if not HOSTLESS_PROTOCOL[proto] and ( + slashes or (proto and not SLASHED_PROTOCOL[proto]) + ): + + # there's a hostname. + # the first instance of /, ?, ;, or # ends the host. + # + # If there is an @ in the hostname, then non-host chars *are* allowed + # to the left of the last @ sign, unless some host-ending character + # comes *before* the @-sign. + # URLs are obnoxious. + # + # ex: + # http://a@b@c/ => user:a@b host:c + # http://a@b?@c => user:a host:c path:/?@c + + # v0.12 TODO(isaacs): This is not quite how Chrome does things. + # Review our test case against browsers more comprehensively. + + # find the first instance of any hostEndingChars + host_end = -1 + for i in range(len(HOST_ENDING_CHARS)): + hec = rest.find(HOST_ENDING_CHARS[i]) + if hec != -1 and (host_end == -1 or hec < host_end): + host_end = hec + + # at this point, either we have an explicit point where the + # auth portion cannot go past, or the last @ char is the decider. + if host_end == -1: + # atSign can be anywhere. + at_sign = rest.rfind("@") + else: + # atSign must be in auth portion. + # http://a@b/c@d => host:b auth:a path:/c@d + at_sign = rest.rfind("@", 0, host_end + 1) + + # Now we have a portion which is definitely the auth. + # Pull that off. + if at_sign != -1: + auth = rest[:at_sign] + rest = rest[at_sign + 1 :] + self.auth = auth + + # the host is the remaining to the left of the first non-host char + host_end = -1 + for i in range(len(NON_HOST_CHARS)): + hec = rest.find(NON_HOST_CHARS[i]) + if hec != -1 and (host_end == -1 or hec < host_end): + host_end = hec + # if we still have not hit it, then the entire thing is a host. + if host_end == -1: + host_end = len(rest) + + if host_end > 0 and rest[host_end - 1] == ":": + host_end -= 1 + host = rest[:host_end] + rest = rest[host_end:] + + # pull out port. + self.parse_host(host) + + # we've indicated that there is a hostname, + # so even if it's empty, it has to be present. + self.hostname = self.hostname or "" + + # if hostname begins with [ and ends with ] + # assume that it's an IPv6 address. + ipv6_hostname = self.hostname.startswith("[") and self.hostname.endswith( + "]" + ) + + # validate a little. + if not ipv6_hostname: + hostparts = self.hostname.split(".") + l = len(hostparts) # noqa: E741 + i = 0 + while i < l: + part = hostparts[i] + if not part: + i += 1 # emulate statement3 in JS for loop + continue + if not HOSTNAME_PART_PATTERN.search(part): + newpart = "" + k = len(part) + j = 0 + while j < k: + if ord(part[j]) > 127: + # we replace non-ASCII char with a temporary placeholder + # we need this to make sure size of hostname is not + # broken by replacing non-ASCII by nothing + newpart += "x" + else: + newpart += part[j] + j += 1 # emulate statement3 in JS for loop + + # we test again with ASCII char only + if not HOSTNAME_PART_PATTERN.search(newpart): + valid_parts = hostparts[:i] + not_host = hostparts[i + 1 :] + bit = HOSTNAME_PART_START.search(part) + if bit: + valid_parts.append(bit.group(1)) + not_host.insert(0, bit.group(2)) + if not_host: + rest = ".".join(not_host) + rest + self.hostname = ".".join(valid_parts) + break + i += 1 # emulate statement3 in JS for loop + + if len(self.hostname) > HOSTNAME_MAX_LEN: + self.hostname = "" + + # strip [ and ] from the hostname + # the host field still retains them, though + if ipv6_hostname: + self.hostname = self.hostname[1:-1] + + # chop off from the tail first. + hash = rest.find("#") # noqa: A001 + if hash != -1: + # got a fragment string. + self.hash = rest[hash:] + rest = rest[:hash] + qm = rest.find("?") + if qm != -1: + self.search = rest[qm:] + rest = rest[:qm] + if rest: + self.pathname = rest + if SLASHED_PROTOCOL[lower_proto] and self.hostname and not self.pathname: + self.pathname = "" + + return self + + def parse_host(self, host: str) -> None: + port_match = PORT_PATTERN.search(host) + if port_match: + port = port_match.group() + if port != ":": + self.port = port[1:] + host = host[: -len(port)] + if host: + self.hostname = host + + +def url_parse(url: URL | str, *, slashes_denote_host: bool = False) -> URL: + if isinstance(url, URL): + return url + u = MutableURL() + u.parse(url, slashes_denote_host) + return URL( + u.protocol, u.slashes, u.auth, u.port, u.hostname, u.hash, u.search, u.pathname + ) diff --git a/lib/python3.1/site-packages/mdurl/_url.py b/lib/python3.1/site-packages/mdurl/_url.py new file mode 100644 index 0000000000000000000000000000000000000000..f866e7a179c8854e37c9bba6294f48681e5d99d7 --- /dev/null +++ b/lib/python3.1/site-packages/mdurl/_url.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from typing import NamedTuple + + +class URL(NamedTuple): + protocol: str | None + slashes: bool + auth: str | None + port: str | None + hostname: str | None + hash: str | None # noqa: A003 + search: str | None + pathname: str | None diff --git a/lib/python3.1/site-packages/mdurl/py.typed b/lib/python3.1/site-packages/mdurl/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..7632ecf77545c5e5501cb3fc5719df0761104ca2 --- /dev/null +++ b/lib/python3.1/site-packages/mdurl/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561 diff --git a/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/INSTALLER b/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/LICENSE b/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..9aae71ed462e08e3e0d955063e7d6679fd38207e --- /dev/null +++ b/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2021, Jim Crist-Harif +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/METADATA b/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..86e47275b0fcee8de01fca85cfbfe999d0473900 --- /dev/null +++ b/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/METADATA @@ -0,0 +1,180 @@ +Metadata-Version: 2.1 +Name: msgspec +Version: 0.19.0 +Summary: A fast serialization and validation library, with builtin support for JSON, MessagePack, YAML, and TOML. +Home-page: https://jcristharif.com/msgspec/ +Maintainer: Jim Crist-Harif +Maintainer-email: jcristharif@gmail.com +License: BSD +Project-URL: Documentation, https://jcristharif.com/msgspec/ +Project-URL: Source, https://github.com/jcrist/msgspec/ +Project-URL: Issue Tracker, https://github.com/jcrist/msgspec/issues +Keywords: JSON msgpack MessagePack TOML YAML serialization validation schema +Classifier: License :: OSI Approved :: BSD License +Classifier: Development Status :: 4 - Beta +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Requires-Python: >=3.9 +Description-Content-Type: text/markdown +License-File: LICENSE +Provides-Extra: yaml +Requires-Dist: pyyaml; extra == "yaml" +Provides-Extra: toml +Requires-Dist: tomli; python_version < "3.11" and extra == "toml" +Requires-Dist: tomli_w; extra == "toml" +Provides-Extra: doc +Requires-Dist: sphinx; extra == "doc" +Requires-Dist: furo; extra == "doc" +Requires-Dist: sphinx-copybutton; extra == "doc" +Requires-Dist: sphinx-design; extra == "doc" +Requires-Dist: ipython; extra == "doc" +Provides-Extra: test +Requires-Dist: pytest; extra == "test" +Requires-Dist: msgpack; extra == "test" +Requires-Dist: attrs; extra == "test" +Requires-Dist: eval-type-backport; python_version < "3.10" and extra == "test" +Requires-Dist: pyyaml; extra == "test" +Requires-Dist: tomli; python_version < "3.11" and extra == "test" +Requires-Dist: tomli_w; extra == "test" +Provides-Extra: dev +Requires-Dist: pre-commit; extra == "dev" +Requires-Dist: coverage; extra == "dev" +Requires-Dist: mypy; extra == "dev" +Requires-Dist: pyright; extra == "dev" +Requires-Dist: sphinx; extra == "dev" +Requires-Dist: furo; extra == "dev" +Requires-Dist: sphinx-copybutton; extra == "dev" +Requires-Dist: sphinx-design; extra == "dev" +Requires-Dist: ipython; extra == "dev" +Requires-Dist: pytest; extra == "dev" +Requires-Dist: msgpack; extra == "dev" +Requires-Dist: attrs; extra == "dev" +Requires-Dist: eval-type-backport; python_version < "3.10" and extra == "dev" +Requires-Dist: pyyaml; extra == "dev" +Requires-Dist: tomli; python_version < "3.11" and extra == "dev" +Requires-Dist: tomli_w; extra == "dev" + +

    + + msgspec + +

    + +

    + + + + + + + + + + + + + + + + + + +

    + + +`msgspec` is a *fast* serialization and validation library, with builtin +support for [JSON](https://json.org), [MessagePack](https://msgpack.org), +[YAML](https://yaml.org), and [TOML](https://toml.io). It features: + +- 🚀 **High performance encoders/decoders** for common protocols. The JSON and + MessagePack implementations regularly + [benchmark](https://jcristharif.com/msgspec/benchmarks.html) as the fastest + options for Python. + +- 🎉 **Support for a wide variety of Python types**. Additional types may be + supported through + [extensions](https://jcristharif.com/msgspec/extending.html). + +- 🔍 **Zero-cost schema validation** using familiar Python type annotations. In + [benchmarks](https://jcristharif.com/msgspec/benchmarks.html) `msgspec` + decodes *and* validates JSON faster than + [orjson](https://github.com/ijl/orjson) can decode it alone. + +- ✨ **A speedy Struct type** for representing structured data. If you already + use [dataclasses](https://docs.python.org/3/library/dataclasses.html) or + [attrs](https://www.attrs.org), + [structs](https://jcristharif.com/msgspec/structs.html) should feel familiar. + However, they're + [5-60x faster](https://jcristharif.com/msgspec/benchmarks.html#benchmark-structs>) + for common operations. + +All of this is included in a +[lightweight library](https://jcristharif.com/msgspec/benchmarks.html#benchmark-library-size) +with no required dependencies. + +--- + +`msgspec` may be used for serialization alone, as a faster JSON or +MessagePack library. For the greatest benefit though, we recommend using +`msgspec` to handle the full serialization & validation workflow: + +**Define** your message schemas using standard Python type annotations. + +```python +>>> import msgspec + +>>> class User(msgspec.Struct): +... """A new type describing a User""" +... name: str +... groups: set[str] = set() +... email: str | None = None +``` + +**Encode** messages as JSON, or one of the many other supported protocols. + +```python +>>> alice = User("alice", groups={"admin", "engineering"}) + +>>> alice +User(name='alice', groups={"admin", "engineering"}, email=None) + +>>> msg = msgspec.json.encode(alice) + +>>> msg +b'{"name":"alice","groups":["admin","engineering"],"email":null}' +``` + +**Decode** messages back into Python objects, with optional schema validation. + +```python +>>> msgspec.json.decode(msg, type=User) +User(name='alice', groups={"admin", "engineering"}, email=None) + +>>> msgspec.json.decode(b'{"name":"bob","groups":[123]}', type=User) +Traceback (most recent call last): + File "", line 1, in +msgspec.ValidationError: Expected `str`, got `int` - at `$.groups[0]` +``` + +`msgspec` is designed to be as performant as possible, while retaining some of +the nicities of validation libraries like +[pydantic](https://pydantic-docs.helpmanual.io/). For supported types, +encoding/decoding a message with `msgspec` can be +[~10-80x faster than alternative libraries](https://jcristharif.com/msgspec/benchmarks.html). + +

    + + + +

    + +See [the documentation](https://jcristharif.com/msgspec/) for more information. + + +## LICENSE + +New BSD. See the +[License File](https://github.com/jcrist/msgspec/blob/main/LICENSE). diff --git a/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/RECORD b/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..0813b32a48958893e43eaa63fad2dd9fcca565c4 --- /dev/null +++ b/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/RECORD @@ -0,0 +1,32 @@ +msgspec-0.19.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +msgspec-0.19.0.dist-info/LICENSE,sha256=b96LZyOIXoUIgI9lrDcv65fT7uLqsrkdNNeWt1t6xzQ,1498 +msgspec-0.19.0.dist-info/METADATA,sha256=8E21mTozMdjXm15-sfmsFJ07AXATjY-a0iQIYCeuJk8,6907 +msgspec-0.19.0.dist-info/RECORD,, +msgspec-0.19.0.dist-info/WHEEL,sha256=m2_gJPr7xnnGfiE013tVzgC-Sc1sFXYskwf7kyqFHXk,151 +msgspec-0.19.0.dist-info/top_level.txt,sha256=8Ysnxa29QLhdPtjPGjyc1hmx3nOmKeu_OAk2682ZdvU,8 +msgspec/__init__.py,sha256=6O0PLIjJk1HSPP3Pu5trzUoUEhKcIxXCqQ7MVa1Ylms,637 +msgspec/__init__.pyi,sha256=A-GkzW1sZ9a-pe-58PzeneEyxJLmbudnBhUA83dYUdw,5566 +msgspec/__pycache__/__init__.cpython-310.pyc,, +msgspec/__pycache__/_json_schema.cpython-310.pyc,, +msgspec/__pycache__/_utils.cpython-310.pyc,, +msgspec/__pycache__/_version.cpython-310.pyc,, +msgspec/__pycache__/inspect.cpython-310.pyc,, +msgspec/__pycache__/json.cpython-310.pyc,, +msgspec/__pycache__/msgpack.cpython-310.pyc,, +msgspec/__pycache__/structs.cpython-310.pyc,, +msgspec/__pycache__/toml.cpython-310.pyc,, +msgspec/__pycache__/yaml.cpython-310.pyc,, +msgspec/_core.cpython-310-x86_64-linux-gnu.so,sha256=1bI4oS04gRuOWIYf99hPhrzhNS-O-pkKwHU82Ko3jko,410256 +msgspec/_json_schema.py,sha256=bKTi8sOiBuNyEUGJhrjnX4rUhYdjK09MtcDgU8mEo1E,16658 +msgspec/_utils.py,sha256=JkY8T6NuCMReOLNNg6lJMrIfqWZ-Ifwj7BBhDNLvD4Q,10101 +msgspec/_version.py,sha256=Dee0CCrWdKy6ga2lPQyuoDgvEhPjYqm6g2dX_XxIxeE,498 +msgspec/inspect.py,sha256=uHY4bwjrfNE4UP6rU1PwaZ3-QeyClw-5hpWyXIuFhq4,28896 +msgspec/json.py,sha256=kIzpm2NHszkAfAbNcemxq_hUHKt3qxH29yABKH0MnTs,212 +msgspec/json.pyi,sha256=nLEHvthIgoMeO_USzf12zcryCZra423VTFGcgdoqB50,3082 +msgspec/msgpack.py,sha256=JSRLcEqpyCgwy_wD1loF3qHN_ytnCQLc8RX33uATbCU,154 +msgspec/msgpack.pyi,sha256=le3E6_MlrUfFTlru6iPSxBOx24WsXAKpOpWGQ85DPQo,2641 +msgspec/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +msgspec/structs.py,sha256=owU3O3w8yi4_lQYAHIRwM0Hre0Oo_puNbz4SM0GP0ew,2942 +msgspec/structs.pyi,sha256=6ob7yIjD6QwMUn_vJHC0CgWgt3CtgRAU2AOUq_1EPGU,902 +msgspec/toml.py,sha256=eXILfJYg9PPkWoJwHyhxoRH8EmuPrmsM2kYfUQH6y8o,5526 +msgspec/yaml.py,sha256=TauuurrHjdm4bb-qIepncZLzg02vse8x4Ju5Rweu8l8,5395 diff --git a/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/WHEEL b/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..f5487b22db738ec9d4f3fcf1afc701af226d4171 --- /dev/null +++ b/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: setuptools (75.6.0) +Root-Is-Purelib: false +Tag: cp310-cp310-manylinux_2_17_x86_64 +Tag: cp310-cp310-manylinux2014_x86_64 + diff --git a/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/top_level.txt b/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..bdc9e5a1e2163cadeda15e89fd084c9ceefcd042 --- /dev/null +++ b/lib/python3.1/site-packages/msgspec-0.19.0.dist-info/top_level.txt @@ -0,0 +1 @@ +msgspec diff --git a/lib/python3.1/site-packages/nest_asyncio.py b/lib/python3.1/site-packages/nest_asyncio.py new file mode 100644 index 0000000000000000000000000000000000000000..1cb5c253fa0658a0adea3516f6463904396cf573 --- /dev/null +++ b/lib/python3.1/site-packages/nest_asyncio.py @@ -0,0 +1,219 @@ +"""Patch asyncio to allow nested event loops.""" + +import asyncio +import asyncio.events as events +import os +import sys +import threading +from contextlib import contextmanager, suppress +from heapq import heappop + + +def apply(loop=None): + """Patch asyncio to make its event loop reentrant.""" + _patch_asyncio() + _patch_policy() + _patch_tornado() + + loop = loop or asyncio.get_event_loop() + _patch_loop(loop) + + +def _patch_asyncio(): + """Patch asyncio module to use pure Python tasks and futures.""" + + def run(main, *, debug=False): + loop = asyncio.get_event_loop() + loop.set_debug(debug) + task = asyncio.ensure_future(main) + try: + return loop.run_until_complete(task) + finally: + if not task.done(): + task.cancel() + with suppress(asyncio.CancelledError): + loop.run_until_complete(task) + + def _get_event_loop(stacklevel=3): + loop = events._get_running_loop() + if loop is None: + loop = events.get_event_loop_policy().get_event_loop() + return loop + + # Use module level _current_tasks, all_tasks and patch run method. + if hasattr(asyncio, '_nest_patched'): + return + if sys.version_info >= (3, 6, 0): + asyncio.Task = asyncio.tasks._CTask = asyncio.tasks.Task = \ + asyncio.tasks._PyTask + asyncio.Future = asyncio.futures._CFuture = asyncio.futures.Future = \ + asyncio.futures._PyFuture + if sys.version_info < (3, 7, 0): + asyncio.tasks._current_tasks = asyncio.tasks.Task._current_tasks + asyncio.all_tasks = asyncio.tasks.Task.all_tasks + if sys.version_info >= (3, 9, 0): + events._get_event_loop = events.get_event_loop = \ + asyncio.get_event_loop = _get_event_loop + asyncio.run = run + asyncio._nest_patched = True + + +def _patch_policy(): + """Patch the policy to always return a patched loop.""" + + def get_event_loop(self): + if self._local._loop is None: + loop = self.new_event_loop() + _patch_loop(loop) + self.set_event_loop(loop) + return self._local._loop + + policy = events.get_event_loop_policy() + policy.__class__.get_event_loop = get_event_loop + + +def _patch_loop(loop): + """Patch loop to make it reentrant.""" + + def run_forever(self): + with manage_run(self), manage_asyncgens(self): + while True: + self._run_once() + if self._stopping: + break + self._stopping = False + + def run_until_complete(self, future): + with manage_run(self): + f = asyncio.ensure_future(future, loop=self) + if f is not future: + f._log_destroy_pending = False + while not f.done(): + self._run_once() + if self._stopping: + break + if not f.done(): + raise RuntimeError( + 'Event loop stopped before Future completed.') + return f.result() + + def _run_once(self): + """ + Simplified re-implementation of asyncio's _run_once that + runs handles as they become ready. + """ + ready = self._ready + scheduled = self._scheduled + while scheduled and scheduled[0]._cancelled: + heappop(scheduled) + + timeout = ( + 0 if ready or self._stopping + else min(max( + scheduled[0]._when - self.time(), 0), 86400) if scheduled + else None) + event_list = self._selector.select(timeout) + self._process_events(event_list) + + end_time = self.time() + self._clock_resolution + while scheduled and scheduled[0]._when < end_time: + handle = heappop(scheduled) + ready.append(handle) + + for _ in range(len(ready)): + if not ready: + break + handle = ready.popleft() + if not handle._cancelled: + # preempt the current task so that that checks in + # Task.__step do not raise + curr_task = curr_tasks.pop(self, None) + + try: + handle._run() + finally: + # restore the current task + if curr_task is not None: + curr_tasks[self] = curr_task + + handle = None + + @contextmanager + def manage_run(self): + """Set up the loop for running.""" + self._check_closed() + old_thread_id = self._thread_id + old_running_loop = events._get_running_loop() + try: + self._thread_id = threading.get_ident() + events._set_running_loop(self) + self._num_runs_pending += 1 + if self._is_proactorloop: + if self._self_reading_future is None: + self.call_soon(self._loop_self_reading) + yield + finally: + self._thread_id = old_thread_id + events._set_running_loop(old_running_loop) + self._num_runs_pending -= 1 + if self._is_proactorloop: + if (self._num_runs_pending == 0 + and self._self_reading_future is not None): + ov = self._self_reading_future._ov + self._self_reading_future.cancel() + if ov is not None: + self._proactor._unregister(ov) + self._self_reading_future = None + + @contextmanager + def manage_asyncgens(self): + if not hasattr(sys, 'get_asyncgen_hooks'): + # Python version is too old. + return + old_agen_hooks = sys.get_asyncgen_hooks() + try: + self._set_coroutine_origin_tracking(self._debug) + if self._asyncgens is not None: + sys.set_asyncgen_hooks( + firstiter=self._asyncgen_firstiter_hook, + finalizer=self._asyncgen_finalizer_hook) + yield + finally: + self._set_coroutine_origin_tracking(False) + if self._asyncgens is not None: + sys.set_asyncgen_hooks(*old_agen_hooks) + + def _check_running(self): + """Do not throw exception if loop is already running.""" + pass + + if hasattr(loop, '_nest_patched'): + return + if not isinstance(loop, asyncio.BaseEventLoop): + raise ValueError('Can\'t patch loop of type %s' % type(loop)) + cls = loop.__class__ + cls.run_forever = run_forever + cls.run_until_complete = run_until_complete + cls._run_once = _run_once + cls._check_running = _check_running + cls._check_runnung = _check_running # typo in Python 3.7 source + cls._num_runs_pending = 1 if loop.is_running() else 0 + cls._is_proactorloop = ( + os.name == 'nt' and issubclass(cls, asyncio.ProactorEventLoop)) + if sys.version_info < (3, 7, 0): + cls._set_coroutine_origin_tracking = cls._set_coroutine_wrapper + curr_tasks = asyncio.tasks._current_tasks \ + if sys.version_info >= (3, 7, 0) else asyncio.Task._current_tasks + cls._nest_patched = True + + +def _patch_tornado(): + """ + If tornado is imported before nest_asyncio, make tornado aware of + the pure-Python asyncio Future. + """ + if 'tornado' in sys.modules: + import tornado.concurrent as tc # type: ignore + tc.Future = asyncio.Future + if asyncio.Future not in tc.FUTURES: + tc.FUTURES += (asyncio.Future,) diff --git a/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/INSTALLER b/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/License.txt b/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/License.txt new file mode 100644 index 0000000000000000000000000000000000000000..b491c70e0aef319022ded661e111ddbd45b8a17f --- /dev/null +++ b/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/License.txt @@ -0,0 +1,1568 @@ +End User License Agreement +-------------------------- + + +Preface +------- + +The Software License Agreement in Chapter 1 and the Supplement +in Chapter 2 contain license terms and conditions that govern +the use of NVIDIA software. By accepting this agreement, you +agree to comply with all the terms and conditions applicable +to the product(s) included herein. + + +NVIDIA Driver + + +Description + +This package contains the operating system driver and +fundamental system software components for NVIDIA GPUs. + + +NVIDIA CUDA Toolkit + + +Description + +The NVIDIA CUDA Toolkit provides command-line and graphical +tools for building, debugging and optimizing the performance +of applications accelerated by NVIDIA GPUs, runtime and math +libraries, and documentation including programming guides, +user manuals, and API references. + + +Default Install Location of CUDA Toolkit + +Windows platform: + +%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v#.# + +Linux platform: + +/usr/local/cuda-#.# + +Mac platform: + +/Developer/NVIDIA/CUDA-#.# + + +NVIDIA CUDA Samples + + +Description + +This package includes over 100+ CUDA examples that demonstrate +various CUDA programming principles, and efficient CUDA +implementation of algorithms in specific application domains. + + +Default Install Location of CUDA Samples + +Windows platform: + +%ProgramData%\NVIDIA Corporation\CUDA Samples\v#.# + +Linux platform: + +/usr/local/cuda-#.#/samples + +and + +$HOME/NVIDIA_CUDA-#.#_Samples + +Mac platform: + +/Developer/NVIDIA/CUDA-#.#/samples + + +NVIDIA Nsight Visual Studio Edition (Windows only) + + +Description + +NVIDIA Nsight Development Platform, Visual Studio Edition is a +development environment integrated into Microsoft Visual +Studio that provides tools for debugging, profiling, analyzing +and optimizing your GPU computing and graphics applications. + + +Default Install Location of Nsight Visual Studio Edition + +Windows platform: + +%ProgramFiles(x86)%\NVIDIA Corporation\Nsight Visual Studio Edition #.# + + +1. License Agreement for NVIDIA Software Development Kits +--------------------------------------------------------- + + +Release Date: July 26, 2018 +--------------------------- + + +Important NoticeRead before downloading, installing, +copying or using the licensed software: +------------------------------------------------------- + +This license agreement, including exhibits attached +("Agreement”) is a legal agreement between you and NVIDIA +Corporation ("NVIDIA") and governs your use of a NVIDIA +software development kit (“SDK”). + +Each SDK has its own set of software and materials, but here +is a description of the types of items that may be included in +a SDK: source code, header files, APIs, data sets and assets +(examples include images, textures, models, scenes, videos, +native API input/output files), binary software, sample code, +libraries, utility programs, programming code and +documentation. + +This Agreement can be accepted only by an adult of legal age +of majority in the country in which the SDK is used. + +If you are entering into this Agreement on behalf of a company +or other legal entity, you represent that you have the legal +authority to bind the entity to this Agreement, in which case +“you” will mean the entity you represent. + +If you don’t have the required age or authority to accept +this Agreement, or if you don’t accept all the terms and +conditions of this Agreement, do not download, install or use +the SDK. + +You agree to use the SDK only for purposes that are permitted +by (a) this Agreement, and (b) any applicable law, regulation +or generally accepted practices or guidelines in the relevant +jurisdictions. + + +1.1. License + + +1.1.1. License Grant + +Subject to the terms of this Agreement, NVIDIA hereby grants +you a non-exclusive, non-transferable license, without the +right to sublicense (except as expressly provided in this +Agreement) to: + + 1. Install and use the SDK, + + 2. Modify and create derivative works of sample source code + delivered in the SDK, and + + 3. Distribute those portions of the SDK that are identified + in this Agreement as distributable, as incorporated in + object code format into a software application that meets + the distribution requirements indicated in this Agreement. + + +1.1.2. Distribution Requirements + +These are the distribution requirements for you to exercise +the distribution grant: + + 1. Your application must have material additional + functionality, beyond the included portions of the SDK. + + 2. The distributable portions of the SDK shall only be + accessed by your application. + + 3. The following notice shall be included in modifications + and derivative works of sample source code distributed: + “This software contains source code provided by NVIDIA + Corporation.” + + 4. Unless a developer tool is identified in this Agreement + as distributable, it is delivered for your internal use + only. + + 5. The terms under which you distribute your application + must be consistent with the terms of this Agreement, + including (without limitation) terms relating to the + license grant and license restrictions and protection of + NVIDIA’s intellectual property rights. Additionally, you + agree that you will protect the privacy, security and + legal rights of your application users. + + 6. You agree to notify NVIDIA in writing of any known or + suspected distribution or use of the SDK not in compliance + with the requirements of this Agreement, and to enforce + the terms of your agreements with respect to distributed + SDK. + + +1.1.3. Authorized Users + +You may allow employees and contractors of your entity or of +your subsidiary(ies) to access and use the SDK from your +secure network to perform work on your behalf. + +If you are an academic institution you may allow users +enrolled or employed by the academic institution to access and +use the SDK from your secure network. + +You are responsible for the compliance with the terms of this +Agreement by your authorized users. If you become aware that +your authorized users didn’t follow the terms of this +Agreement, you agree to take reasonable steps to resolve the +non-compliance and prevent new occurrences. + + +1.1.4. Pre-Release SDK + +The SDK versions identified as alpha, beta, preview or +otherwise as pre-release, may not be fully functional, may +contain errors or design flaws, and may have reduced or +different security, privacy, accessibility, availability, and +reliability standards relative to commercial versions of +NVIDIA software and materials. Use of a pre-release SDK may +result in unexpected results, loss of data, project delays or +other unpredictable damage or loss. + +You may use a pre-release SDK at your own risk, understanding +that pre-release SDKs are not intended for use in production +or business-critical systems. + +NVIDIA may choose not to make available a commercial version +of any pre-release SDK. NVIDIA may also choose to abandon +development and terminate the availability of a pre-release +SDK at any time without liability. + + +1.1.5. Updates + +NVIDIA may, at its option, make available patches, workarounds +or other updates to this SDK. Unless the updates are provided +with their separate governing terms, they are deemed part of +the SDK licensed to you as provided in this Agreement. You +agree that the form and content of the SDK that NVIDIA +provides may change without prior notice to you. While NVIDIA +generally maintains compatibility between versions, NVIDIA may +in some cases make changes that introduce incompatibilities in +future versions of the SDK. + + +1.1.6. Third Party Licenses + +The SDK may come bundled with, or otherwise include or be +distributed with, third party software licensed by a NVIDIA +supplier and/or open source software provided under an open +source license. Use of third party software is subject to the +third-party license terms, or in the absence of third party +terms, the terms of this Agreement. Copyright to third party +software is held by the copyright holders indicated in the +third-party software or license. + + +1.1.7. Reservation of Rights + +NVIDIA reserves all rights, title, and interest in and to the +SDK, not expressly granted to you under this Agreement. + + +1.2. Limitations + +The following license limitations apply to your use of the +SDK: + + 1. You may not reverse engineer, decompile or disassemble, + or remove copyright or other proprietary notices from any + portion of the SDK or copies of the SDK. + + 2. Except as expressly provided in this Agreement, you may + not copy, sell, rent, sublicense, transfer, distribute, + modify, or create derivative works of any portion of the + SDK. For clarity, you may not distribute or sublicense the + SDK as a stand-alone product. + + 3. Unless you have an agreement with NVIDIA for this + purpose, you may not indicate that an application created + with the SDK is sponsored or endorsed by NVIDIA. + + 4. You may not bypass, disable, or circumvent any + encryption, security, digital rights management or + authentication mechanism in the SDK. + + 5. You may not use the SDK in any manner that would cause it + to become subject to an open source software license. As + examples, licenses that require as a condition of use, + modification, and/or distribution that the SDK be: + + a. Disclosed or distributed in source code form; + + b. Licensed for the purpose of making derivative works; + or + + c. Redistributable at no charge. + + 6. Unless you have an agreement with NVIDIA for this + purpose, you may not use the SDK with any system or + application where the use or failure of the system or + application can reasonably be expected to threaten or + result in personal injury, death, or catastrophic loss. + Examples include use in avionics, navigation, military, + medical, life support or other life critical applications. + NVIDIA does not design, test or manufacture the SDK for + these critical uses and NVIDIA shall not be liable to you + or any third party, in whole or in part, for any claims or + damages arising from such uses. + + 7. You agree to defend, indemnify and hold harmless NVIDIA + and its affiliates, and their respective employees, + contractors, agents, officers and directors, from and + against any and all claims, damages, obligations, losses, + liabilities, costs or debt, fines, restitutions and + expenses (including but not limited to attorney’s fees + and costs incident to establishing the right of + indemnification) arising out of or related to your use of + the SDK outside of the scope of this Agreement, or not in + compliance with its terms. + + +1.3. Ownership + + 1. NVIDIA or its licensors hold all rights, title and + interest in and to the SDK and its modifications and + derivative works, including their respective intellectual + property rights, subject to your rights described in this + section. This SDK may include software and materials from + NVIDIA’s licensors, and these licensors are intended + third party beneficiaries that may enforce this Agreement + with respect to their intellectual property rights. + + 2. You hold all rights, title and interest in and to your + applications and your derivative works of the sample + source code delivered in the SDK, including their + respective intellectual property rights, subject to + NVIDIA’s rights described in this section. + + 3. You may, but don’t have to, provide to NVIDIA + suggestions, feature requests or other feedback regarding + the SDK, including possible enhancements or modifications + to the SDK. For any feedback that you voluntarily provide, + you hereby grant NVIDIA and its affiliates a perpetual, + non-exclusive, worldwide, irrevocable license to use, + reproduce, modify, license, sublicense (through multiple + tiers of sublicensees), and distribute (through multiple + tiers of distributors) it without the payment of any + royalties or fees to you. NVIDIA will use feedback at its + choice. NVIDIA is constantly looking for ways to improve + its products, so you may send feedback to NVIDIA through + the developer portal at https://developer.nvidia.com. + + +1.4. No Warranties + +THE SDK IS PROVIDED BY NVIDIA “AS IS” AND “WITH ALL +FAULTS.” TO THE MAXIMUM EXTENT PERMITTED BY LAW, NVIDIA AND +ITS AFFILIATES EXPRESSLY DISCLAIM ALL WARRANTIES OF ANY KIND +OR NATURE, WHETHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING, +BUT NOT LIMITED TO, ANY WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE, TITLE, NON-INFRINGEMENT, OR THE +ABSENCE OF ANY DEFECTS THEREIN, WHETHER LATENT OR PATENT. NO +WARRANTY IS MADE ON THE BASIS OF TRADE USAGE, COURSE OF +DEALING OR COURSE OF TRADE. + + +1.5. Limitation of Liability + +TO THE MAXIMUM EXTENT PERMITTED BY LAW, NVIDIA AND ITS +AFFILIATES SHALL NOT BE LIABLE FOR ANY SPECIAL, INCIDENTAL, +PUNITIVE OR CONSEQUENTIAL DAMAGES, OR ANY LOST PROFITS, LOSS +OF USE, LOSS OF DATA OR LOSS OF GOODWILL, OR THE COSTS OF +PROCURING SUBSTITUTE PRODUCTS, ARISING OUT OF OR IN CONNECTION +WITH THIS AGREEMENT OR THE USE OR PERFORMANCE OF THE SDK, +WHETHER SUCH LIABILITY ARISES FROM ANY CLAIM BASED UPON BREACH +OF CONTRACT, BREACH OF WARRANTY, TORT (INCLUDING NEGLIGENCE), +PRODUCT LIABILITY OR ANY OTHER CAUSE OF ACTION OR THEORY OF +LIABILITY. IN NO EVENT WILL NVIDIA’S AND ITS AFFILIATES +TOTAL CUMULATIVE LIABILITY UNDER OR ARISING OUT OF THIS +AGREEMENT EXCEED US$10.00. THE NATURE OF THE LIABILITY OR THE +NUMBER OF CLAIMS OR SUITS SHALL NOT ENLARGE OR EXTEND THIS +LIMIT. + +These exclusions and limitations of liability shall apply +regardless if NVIDIA or its affiliates have been advised of +the possibility of such damages, and regardless of whether a +remedy fails its essential purpose. These exclusions and +limitations of liability form an essential basis of the +bargain between the parties, and, absent any of these +exclusions or limitations of liability, the provisions of this +Agreement, including, without limitation, the economic terms, +would be substantially different. + + +1.6. Termination + + 1. This Agreement will continue to apply until terminated by + either you or NVIDIA as described below. + + 2. If you want to terminate this Agreement, you may do so by + stopping to use the SDK. + + 3. NVIDIA may, at any time, terminate this Agreement if: + + a. (i) you fail to comply with any term of this + Agreement and the non-compliance is not fixed within + thirty (30) days following notice from NVIDIA (or + immediately if you violate NVIDIA’s intellectual + property rights); + + b. (ii) you commence or participate in any legal + proceeding against NVIDIA with respect to the SDK; or + + c. (iii) NVIDIA decides to no longer provide the SDK in + a country or, in NVIDIA’s sole discretion, the + continued use of it is no longer commercially viable. + + 4. Upon any termination of this Agreement, you agree to + promptly discontinue use of the SDK and destroy all copies + in your possession or control. Your prior distributions in + accordance with this Agreement are not affected by the + termination of this Agreement. Upon written request, you + will certify in writing that you have complied with your + commitments under this section. Upon any termination of + this Agreement all provisions survive except for the + license grant provisions. + + +1.7. General + +If you wish to assign this Agreement or your rights and +obligations, including by merger, consolidation, dissolution +or operation of law, contact NVIDIA to ask for permission. Any +attempted assignment not approved by NVIDIA in writing shall +be void and of no effect. NVIDIA may assign, delegate or +transfer this Agreement and its rights and obligations, and if +to a non-affiliate you will be notified. + +You agree to cooperate with NVIDIA and provide reasonably +requested information to verify your compliance with this +Agreement. + +This Agreement will be governed in all respects by the laws of +the United States and of the State of Delaware as those laws +are applied to contracts entered into and performed entirely +within Delaware by Delaware residents, without regard to the +conflicts of laws principles. The United Nations Convention on +Contracts for the International Sale of Goods is specifically +disclaimed. You agree to all terms of this Agreement in the +English language. + +The state or federal courts residing in Santa Clara County, +California shall have exclusive jurisdiction over any dispute +or claim arising out of this Agreement. Notwithstanding this, +you agree that NVIDIA shall still be allowed to apply for +injunctive remedies or an equivalent type of urgent legal +relief in any jurisdiction. + +If any court of competent jurisdiction determines that any +provision of this Agreement is illegal, invalid or +unenforceable, such provision will be construed as limited to +the extent necessary to be consistent with and fully +enforceable under the law and the remaining provisions will +remain in full force and effect. Unless otherwise specified, +remedies are cumulative. + +Each party acknowledges and agrees that the other is an +independent contractor in the performance of this Agreement. + +The SDK has been developed entirely at private expense and is +“commercial items” consisting of “commercial computer +software” and “commercial computer software +documentation” provided with RESTRICTED RIGHTS. Use, +duplication or disclosure by the U.S. Government or a U.S. +Government subcontractor is subject to the restrictions in +this Agreement pursuant to DFARS 227.7202-3(a) or as set forth +in subparagraphs (c)(1) and (2) of the Commercial Computer +Software - Restricted Rights clause at FAR 52.227-19, as +applicable. Contractor/manufacturer is NVIDIA, 2788 San Tomas +Expressway, Santa Clara, CA 95051. + +The SDK is subject to United States export laws and +regulations. You agree that you will not ship, transfer or +export the SDK into any country, or use the SDK in any manner, +prohibited by the United States Bureau of Industry and +Security or economic sanctions regulations administered by the +U.S. Department of Treasury’s Office of Foreign Assets +Control (OFAC), or any applicable export laws, restrictions or +regulations. These laws include restrictions on destinations, +end users and end use. By accepting this Agreement, you +confirm that you are not a resident or citizen of any country +currently embargoed by the U.S. and that you are not otherwise +prohibited from receiving the SDK. + +Any notice delivered by NVIDIA to you under this Agreement +will be delivered via mail, email or fax. You agree that any +notices that NVIDIA sends you electronically will satisfy any +legal communication requirements. Please direct your legal +notices or other correspondence to NVIDIA Corporation, 2788 +San Tomas Expressway, Santa Clara, California 95051, United +States of America, Attention: Legal Department. + +This Agreement and any exhibits incorporated into this +Agreement constitute the entire agreement of the parties with +respect to the subject matter of this Agreement and supersede +all prior negotiations or documentation exchanged between the +parties relating to this SDK license. Any additional and/or +conflicting terms on documents issued by you are null, void, +and invalid. Any amendment or waiver under this Agreement +shall be in writing and signed by representatives of both +parties. + + +2. CUDA Toolkit Supplement to Software License Agreement for +NVIDIA Software Development Kits +------------------------------------------------------------ + + +Release date: August 16, 2018 +----------------------------- + +The terms in this supplement govern your use of the NVIDIA +CUDA Toolkit SDK under the terms of your license agreement +(“Agreement”) as modified by this supplement. Capitalized +terms used but not defined below have the meaning assigned to +them in the Agreement. + +This supplement is an exhibit to the Agreement and is +incorporated as an integral part of the Agreement. In the +event of conflict between the terms in this supplement and the +terms in the Agreement, the terms in this supplement govern. + + +2.1. License Scope + +The SDK is licensed for you to develop applications only for +use in systems with NVIDIA GPUs. + + +2.2. Distribution + +The portions of the SDK that are distributable under the +Agreement are listed in Attachment A. + + +2.3. Operating Systems + +Those portions of the SDK designed exclusively for use on the +Linux or FreeBSD operating systems, or other operating systems +derived from the source code to these operating systems, may +be copied and redistributed for use in accordance with this +Agreement, provided that the object code files are not +modified in any way (except for unzipping of compressed +files). + + +2.4. Audio and Video Encoders and Decoders + +You acknowledge and agree that it is your sole responsibility +to obtain any additional third-party licenses required to +make, have made, use, have used, sell, import, and offer for +sale your products or services that include or incorporate any +third-party software and content relating to audio and/or +video encoders and decoders from, including but not limited +to, Microsoft, Thomson, Fraunhofer IIS, Sisvel S.p.A., +MPEG-LA, and Coding Technologies. NVIDIA does not grant to you +under this Agreement any necessary patent or other rights with +respect to any audio and/or video encoders and decoders. + + +2.5. Licensing + +If the distribution terms in this Agreement are not suitable +for your organization, or for any questions regarding this +Agreement, please contact NVIDIA at +nvidia-compute-license-questions@nvidia.com. + + +2.6. Attachment A + +The following portions of the SDK are distributable under the +Agreement: + +Component + +CUDA Runtime + +Windows + +cudart.dll, cudart_static.lib, cudadevrt.lib + +Mac OSX + +libcudart.dylib, libcudart_static.a, libcudadevrt.a + +Linux + +libcudart.so, libcudart_static.a, libcudadevrt.a + +Android + +libcudart.so, libcudart_static.a, libcudadevrt.a + +Component + +CUDA FFT Library + +Windows + +cufft.dll, cufftw.dll, cufft.lib, cufftw.lib + +Mac OSX + +libcufft.dylib, libcufft_static.a, libcufftw.dylib, +libcufftw_static.a + +Linux + +libcufft.so, libcufft_static.a, libcufftw.so, +libcufftw_static.a + +Android + +libcufft.so, libcufft_static.a, libcufftw.so, +libcufftw_static.a + +Component + +CUDA BLAS Library + +Windows + +cublas.dll, cublasLt.dll + +Mac OSX + +libcublas.dylib, libcublasLt.dylib, libcublas_static.a, +libcublasLt_static.a + +Linux + +libcublas.so, libcublasLt.so, libcublas_static.a, +libcublasLt_static.a + +Android + +libcublas.so, libcublasLt.so, libcublas_static.a, +libcublasLt_static.a + +Component + +NVIDIA "Drop-in" BLAS Library + +Windows + +nvblas.dll + +Mac OSX + +libnvblas.dylib + +Linux + +libnvblas.so + +Component + +CUDA Sparse Matrix Library + +Windows + +cusparse.dll, cusparse.lib + +Mac OSX + +libcusparse.dylib, libcusparse_static.a + +Linux + +libcusparse.so, libcusparse_static.a + +Android + +libcusparse.so, libcusparse_static.a + +Component + +CUDA Linear Solver Library + +Windows + +cusolver.dll, cusolver.lib + +Mac OSX + +libcusolver.dylib, libcusolver_static.a + +Linux + +libcusolver.so, libcusolver_static.a + +Android + +libcusolver.so, libcusolver_static.a + +Component + +CUDA Random Number Generation Library + +Windows + +curand.dll, curand.lib + +Mac OSX + +libcurand.dylib, libcurand_static.a + +Linux + +libcurand.so, libcurand_static.a + +Android + +libcurand.so, libcurand_static.a + +Component + +CUDA Accelerated Graph Library + +Component + +NVIDIA Performance Primitives Library + +Windows + +nppc.dll, nppc.lib, nppial.dll, nppial.lib, nppicc.dll, +nppicc.lib, nppicom.dll, nppicom.lib, nppidei.dll, +nppidei.lib, nppif.dll, nppif.lib, nppig.dll, nppig.lib, +nppim.dll, nppim.lib, nppist.dll, nppist.lib, nppisu.dll, +nppisu.lib, nppitc.dll, nppitc.lib, npps.dll, npps.lib + +Mac OSX + +libnppc.dylib, libnppc_static.a, libnppial.dylib, +libnppial_static.a, libnppicc.dylib, libnppicc_static.a, +libnppicom.dylib, libnppicom_static.a, libnppidei.dylib, +libnppidei_static.a, libnppif.dylib, libnppif_static.a, +libnppig.dylib, libnppig_static.a, libnppim.dylib, +libnppisu_static.a, libnppitc.dylib, libnppitc_static.a, +libnpps.dylib, libnpps_static.a + +Linux + +libnppc.so, libnppc_static.a, libnppial.so, +libnppial_static.a, libnppicc.so, libnppicc_static.a, +libnppicom.so, libnppicom_static.a, libnppidei.so, +libnppidei_static.a, libnppif.so, libnppif_static.a +libnppig.so, libnppig_static.a, libnppim.so, +libnppim_static.a, libnppist.so, libnppist_static.a, +libnppisu.so, libnppisu_static.a, libnppitc.so +libnppitc_static.a, libnpps.so, libnpps_static.a + +Android + +libnppc.so, libnppc_static.a, libnppial.so, +libnppial_static.a, libnppicc.so, libnppicc_static.a, +libnppicom.so, libnppicom_static.a, libnppidei.so, +libnppidei_static.a, libnppif.so, libnppif_static.a +libnppig.so, libnppig_static.a, libnppim.so, +libnppim_static.a, libnppist.so, libnppist_static.a, +libnppisu.so, libnppisu_static.a, libnppitc.so +libnppitc_static.a, libnpps.so, libnpps_static.a + +Component + +NVIDIA JPEG Library + +Linux + +libnvjpeg.so, libnvjpeg_static.a + +Component + +Internal common library required for statically linking to +cuBLAS, cuSPARSE, cuFFT, cuRAND, nvJPEG and NPP + +Mac OSX + +libculibos.a + +Linux + +libculibos.a + +Component + +NVIDIA Runtime Compilation Library and Header + +All + +nvrtc.h + +Windows + +nvrtc.dll, nvrtc-builtins.dll + +Mac OSX + +libnvrtc.dylib, libnvrtc-builtins.dylib + +Linux + +libnvrtc.so, libnvrtc-builtins.so + +Component + +NVIDIA Optimizing Compiler Library + +Windows + +nvvm.dll + +Mac OSX + +libnvvm.dylib + +Linux + +libnvvm.so + +Component + +NVIDIA Common Device Math Functions Library + +Windows + +libdevice.10.bc + +Mac OSX + +libdevice.10.bc + +Linux + +libdevice.10.bc + +Component + +CUDA Occupancy Calculation Header Library + +All + +cuda_occupancy.h + +Component + +CUDA Half Precision Headers + +All + +cuda_fp16.h, cuda_fp16.hpp + +Component + +CUDA Profiling Tools Interface (CUPTI) Library + +Windows + +cupti.dll + +Mac OSX + +libcupti.dylib + +Linux + +libcupti.so + +Component + +NVIDIA Tools Extension Library + +Windows + +nvToolsExt.dll, nvToolsExt.lib + +Mac OSX + +libnvToolsExt.dylib + +Linux + +libnvToolsExt.so + +Component + +NVIDIA CUDA Driver Libraries + +Linux + +libcuda.so, libnvidia-fatbinaryloader.so, +libnvidia-ptxjitcompiler.so + +The NVIDIA CUDA Driver Libraries are only distributable in +applications that meet this criteria: + + 1. The application was developed starting from a NVIDIA CUDA + container obtained from Docker Hub or the NVIDIA GPU + Cloud, and + + 2. The resulting application is packaged as a Docker + container and distributed to users on Docker Hub or the + NVIDIA GPU Cloud only. + + +2.7. Attachment B + + +Additional Licensing Obligations + +The following third party components included in the SOFTWARE +are licensed to Licensee pursuant to the following terms and +conditions: + + 1. Licensee's use of the GDB third party component is + subject to the terms and conditions of GNU GPL v3: + + This product includes copyrighted third-party software licensed + under the terms of the GNU General Public License v3 ("GPL v3"). + All third-party software packages are copyright by their respective + authors. GPL v3 terms and conditions are hereby incorporated into + the Agreement by this reference: http://www.gnu.org/licenses/gpl.txt + + Consistent with these licensing requirements, the software + listed below is provided under the terms of the specified + open source software licenses. To obtain source code for + software provided under licenses that require + redistribution of source code, including the GNU General + Public License (GPL) and GNU Lesser General Public License + (LGPL), contact oss-requests@nvidia.com. This offer is + valid for a period of three (3) years from the date of the + distribution of this product by NVIDIA CORPORATION. + + Component License + CUDA-GDB GPL v3 + + 2. Licensee represents and warrants that any and all third + party licensing and/or royalty payment obligations in + connection with Licensee's use of the H.264 video codecs + are solely the responsibility of Licensee. + + 3. Licensee's use of the Thrust library is subject to the + terms and conditions of the Apache License Version 2.0. + All third-party software packages are copyright by their + respective authors. Apache License Version 2.0 terms and + conditions are hereby incorporated into the Agreement by + this reference. + http://www.apache.org/licenses/LICENSE-2.0.html + + In addition, Licensee acknowledges the following notice: + Thrust includes source code from the Boost Iterator, + Tuple, System, and Random Number libraries. + + Boost Software License - Version 1.0 - August 17th, 2003 + . . . . + + Permission is hereby granted, free of charge, to any person or + organization obtaining a copy of the software and accompanying + documentation covered by this license (the "Software") to use, + reproduce, display, distribute, execute, and transmit the Software, + and to prepare derivative works of the Software, and to permit + third-parties to whom the Software is furnished to do so, all + subject to the following: + + The copyright notices in the Software and this entire statement, + including the above license grant, this restriction and the following + disclaimer, must be included in all copies of the Software, in whole + or in part, and all derivative works of the Software, unless such + copies or derivative works are solely in the form of machine-executable + object code generated by a source language processor. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE AND + NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR + ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR ANY DAMAGES OR + OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + 4. Licensee's use of the LLVM third party component is + subject to the following terms and conditions: + + ====================================================== + LLVM Release License + ====================================================== + University of Illinois/NCSA + Open Source License + + Copyright (c) 2003-2010 University of Illinois at Urbana-Champaign. + All rights reserved. + + Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to + deal with the Software without restriction, including without limitation the + rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at Urbana- + Champaign, nor the names of its contributors may be used to endorse or + promote products derived from this Software without specific prior + written permission. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS WITH THE SOFTWARE. + + 5. Licensee's use (e.g. nvprof) of the PCRE third party + component is subject to the following terms and + conditions: + + ------------ + PCRE LICENCE + ------------ + PCRE is a library of functions to support regular expressions whose syntax + and semantics are as close as possible to those of the Perl 5 language. + Release 8 of PCRE is distributed under the terms of the "BSD" licence, as + specified below. The documentation for PCRE, supplied in the "doc" + directory, is distributed under the same terms as the software itself. The + basic library functions are written in C and are freestanding. Also + included in the distribution is a set of C++ wrapper functions, and a just- + in-time compiler that can be used to optimize pattern matching. These are + both optional features that can be omitted when the library is built. + + THE BASIC LIBRARY FUNCTIONS + --------------------------- + Written by: Philip Hazel + Email local part: ph10 + Email domain: cam.ac.uk + University of Cambridge Computing Service, + Cambridge, England. + Copyright (c) 1997-2012 University of Cambridge + All rights reserved. + + PCRE JUST-IN-TIME COMPILATION SUPPORT + ------------------------------------- + Written by: Zoltan Herczeg + Email local part: hzmester + Emain domain: freemail.hu + Copyright(c) 2010-2012 Zoltan Herczeg + All rights reserved. + + STACK-LESS JUST-IN-TIME COMPILER + -------------------------------- + Written by: Zoltan Herczeg + Email local part: hzmester + Emain domain: freemail.hu + Copyright(c) 2009-2012 Zoltan Herczeg + All rights reserved. + + THE C++ WRAPPER FUNCTIONS + ------------------------- + Contributed by: Google Inc. + Copyright (c) 2007-2012, Google Inc. + All rights reserved. + + THE "BSD" LICENCE + ----------------- + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + 6. Some of the cuBLAS library routines were written by or + derived from code written by Vasily Volkov and are subject + to the Modified Berkeley Software Distribution License as + follows: + + Copyright (c) 2007-2009, Regents of the University of California + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of the University of California, Berkeley nor + the names of its contributors may be used to endorse or promote + products derived from this software without specific prior + written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + 7. Some of the cuBLAS library routines were written by or + derived from code written by Davide Barbieri and are + subject to the Modified Berkeley Software Distribution + License as follows: + + Copyright (c) 2008-2009 Davide Barbieri @ University of Rome Tor Vergata. + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * The name of the author may not be used to endorse or promote + products derived from this software without specific prior + written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + 8. Some of the cuBLAS library routines were derived from + code developed by the University of Tennessee and are + subject to the Modified Berkeley Software Distribution + License as follows: + + Copyright (c) 2010 The University of Tennessee. + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer listed in this license in the documentation and/or + other materials provided with the distribution. + * Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 9. Some of the cuBLAS library routines were written by or + derived from code written by Jonathan Hogg and are subject + to the Modified Berkeley Software Distribution License as + follows: + + Copyright (c) 2012, The Science and Technology Facilities Council (STFC). + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of the STFC nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE STFC BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 10. Some of the cuBLAS library routines were written by or + derived from code written by Ahmad M. Abdelfattah, David + Keyes, and Hatem Ltaief, and are subject to the Apache + License, Version 2.0, as follows: + + -- (C) Copyright 2013 King Abdullah University of Science and Technology + Authors: + Ahmad Abdelfattah (ahmad.ahmad@kaust.edu.sa) + David Keyes (david.keyes@kaust.edu.sa) + Hatem Ltaief (hatem.ltaief@kaust.edu.sa) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the King Abdullah University of Science and + Technology nor the names of its contributors may be used to endorse + or promote products derived from this software without specific prior + written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE + + 11. Some of the cuSPARSE library routines were written by or + derived from code written by Li-Wen Chang and are subject + to the NCSA Open Source License as follows: + + Copyright (c) 2012, University of Illinois. + + All rights reserved. + + Developed by: IMPACT Group, University of Illinois, http://impact.crhc.illinois.edu + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal with the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimers in the documentation and/or other materials provided + with the distribution. + * Neither the names of IMPACT Group, University of Illinois, nor + the names of its contributors may be used to endorse or promote + products derived from this Software without specific prior + written permission. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE + SOFTWARE. + + 12. Some of the cuRAND library routines were written by or + derived from code written by Mutsuo Saito and Makoto + Matsumoto and are subject to the following license: + + Copyright (c) 2009, 2010 Mutsuo Saito, Makoto Matsumoto and Hiroshima + University. All rights reserved. + + Copyright (c) 2011 Mutsuo Saito, Makoto Matsumoto, Hiroshima + University and University of Tokyo. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of the Hiroshima University nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 13. Some of the cuRAND library routines were derived from + code developed by D. E. Shaw Research and are subject to + the following license: + + Copyright 2010-2011, D. E. Shaw Research. + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions, and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions, and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of D. E. Shaw Research nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 14. Some of the Math library routines were written by or + derived from code developed by Norbert Juffa and are + subject to the following license: + + Copyright (c) 2015-2017, Norbert Juffa + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 15. Licensee's use of the lz4 third party component is + subject to the following terms and conditions: + + Copyright (C) 2011-2013, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 16. The NPP library uses code from the Boost Math Toolkit, + and is subject to the following license: + + Boost Software License - Version 1.0 - August 17th, 2003 + . . . . + + Permission is hereby granted, free of charge, to any person or + organization obtaining a copy of the software and accompanying + documentation covered by this license (the "Software") to use, + reproduce, display, distribute, execute, and transmit the Software, + and to prepare derivative works of the Software, and to permit + third-parties to whom the Software is furnished to do so, all + subject to the following: + + The copyright notices in the Software and this entire statement, + including the above license grant, this restriction and the following + disclaimer, must be included in all copies of the Software, in whole + or in part, and all derivative works of the Software, unless such + copies or derivative works are solely in the form of machine-executable + object code generated by a source language processor. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE AND + NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR + ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR ANY DAMAGES OR + OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + 17. Portions of the Nsight Eclipse Edition is subject to the + following license: + + The Eclipse Foundation makes available all content in this plug-in + ("Content"). Unless otherwise indicated below, the Content is provided + to you under the terms and conditions of the Eclipse Public License + Version 1.0 ("EPL"). A copy of the EPL is available at http:// + www.eclipse.org/legal/epl-v10.html. For purposes of the EPL, "Program" + will mean the Content. + + If you did not receive this Content directly from the Eclipse + Foundation, the Content is being redistributed by another party + ("Redistributor") and different terms and conditions may apply to your + use of any object code in the Content. Check the Redistributor's + license that was provided with the Content. If no such license exists, + contact the Redistributor. Unless otherwise indicated below, the terms + and conditions of the EPL still apply to any source code in the + Content and such source code may be obtained at http://www.eclipse.org. + + 18. Some of the cuBLAS library routines uses code from + OpenAI, which is subject to the following license: + + License URL + https://github.com/openai/openai-gemm/blob/master/LICENSE + + License Text + The MIT License + + Copyright (c) 2016 OpenAI (http://openai.com), 2016 Google Inc. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + + 19. Licensee's use of the Visual Studio Setup Configuration + Samples is subject to the following license: + + The MIT License (MIT) + Copyright (C) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + 20. Licensee's use of linmath.h header for CPU functions for + GL vector/matrix operations from lunarG is subject to the + Apache License Version 2.0. + + 21. The DX12-CUDA sample uses the d3dx12.h header, which is + subject to the MIT license . + +----------------- diff --git a/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/METADATA b/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..ea13e81c522e89bc486965b1c722401fb25d8ae5 --- /dev/null +++ b/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/METADATA @@ -0,0 +1,35 @@ +Metadata-Version: 2.1 +Name: nvidia-curand-cu12 +Version: 10.3.5.147 +Summary: CURAND native runtime libraries +Home-page: https://developer.nvidia.com/cuda-zone +Author: Nvidia CUDA Installer Team +Author-email: cuda_installer@nvidia.com +License: NVIDIA Proprietary Software +Keywords: cuda,nvidia,runtime,machine learning,deep learning +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Education +Classifier: Intended Audience :: Science/Research +Classifier: License :: Other/Proprietary License +Classifier: Natural Language :: English +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Topic :: Scientific/Engineering +Classifier: Topic :: Scientific/Engineering :: Mathematics +Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence +Classifier: Topic :: Software Development +Classifier: Topic :: Software Development :: Libraries +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX :: Linux +Requires-Python: >=3 +License-File: License.txt + +CURAND native runtime libraries diff --git a/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/RECORD b/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..5f7de8c02ea54fb928045e378ee2aa0232d00eee --- /dev/null +++ b/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/RECORD @@ -0,0 +1,32 @@ +nvidia/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +nvidia/__pycache__/__init__.cpython-310.pyc,, +nvidia/curand/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +nvidia/curand/__pycache__/__init__.cpython-310.pyc,, +nvidia/curand/include/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +nvidia/curand/include/__pycache__/__init__.cpython-310.pyc,, +nvidia/curand/include/curand.h,sha256=PSwpaDTgFZ4X95KNhip35vChk_nWdB4r086wK1XHqYs,43966 +nvidia/curand/include/curand_discrete.h,sha256=2qD3BkI622XEu0444wVP7HeYkKAx0Rjr2HDhqU4SA7E,3486 +nvidia/curand/include/curand_discrete2.h,sha256=ZrQTO5R9x83AMX88uq7M8M94DLSC5VEz0PAkfcwtQeg,10883 +nvidia/curand/include/curand_globals.h,sha256=bES1Kx0NrATXk1DReMMkqWrB062nOnaAp39y22wViXU,3717 +nvidia/curand/include/curand_kernel.h,sha256=SjfAeh13ybXIxiekcgczzua02kIAqETopJKRhYvCat8,53133 +nvidia/curand/include/curand_lognormal.h,sha256=-X-iNkJSzWpAYYjogm689EJTZfzore9sxU7ObddljLk,28142 +nvidia/curand/include/curand_mrg32k3a.h,sha256=ZVVREjGNsJQJ-3IzZZ_LKGtGteslicb8E0Aly49BKPs,170296 +nvidia/curand/include/curand_mtgp32.h,sha256=Qhrmx0pHWF-P2Uu5bKwYE9ymEWq3c7qBzCITVMaKMfI,7845 +nvidia/curand/include/curand_mtgp32_host.h,sha256=SXqzmSQkzTLSRJ4pojTg_TNCC3T-G89HdBK-boSDqr4,18274 +nvidia/curand/include/curand_mtgp32_kernel.h,sha256=ajZnXr5ZXnQExElf6LPpigrrKPTmMIZbRyTEnJ-BDhw,13731 +nvidia/curand/include/curand_mtgp32dc_p_11213.h,sha256=7_gGYUH47UugIAEt60vYH5nFa-QUwTpDwSEgLg9cZts,276889 +nvidia/curand/include/curand_normal.h,sha256=lnmYVk2fn0oEVWOytdKhXrHL36GLCjMnB8OnZeCaYcA,26953 +nvidia/curand/include/curand_normal_static.h,sha256=5K4iTC9AuSWCe1LVxuj_0y3BVjtp0bxO6hndv2rbmiw,4727 +nvidia/curand/include/curand_philox4x32_x.h,sha256=T21IP-Rdg3_tSVU9Je4dLKuwEqE4ovfwi7r1hOY92Dw,7166 +nvidia/curand/include/curand_poisson.h,sha256=KrhXOmO_D7aclnj8geIyHqdpSQwWHurS9V_pVtgzodM,25461 +nvidia/curand/include/curand_precalc.h,sha256=I6NZdgT42fMm9qSCtP-rlOAqt4Zsqgal0ajktcPmEak,1392393 +nvidia/curand/include/curand_uniform.h,sha256=gpmRgQu5r6ppgLTg60NXoDdVJS6wMUy6jC5bh8l04e8,17472 +nvidia/curand/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +nvidia/curand/lib/__pycache__/__init__.cpython-310.pyc,, +nvidia/curand/lib/libcurand.so.10,sha256=2rgHS2ELgqhjpC7O2niOmwg2S1Rbq5SFCTBrSMRgGM8,96525744 +nvidia_curand_cu12-10.3.5.147.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +nvidia_curand_cu12-10.3.5.147.dist-info/License.txt,sha256=rW9YU_ugyg0VnQ9Y1JrkmDDC-Mk_epJki5zpCttMbM0,59262 +nvidia_curand_cu12-10.3.5.147.dist-info/METADATA,sha256=dJa5-rFt0tbZqHTScSynDoxiJxHTkFBVBtliv7bhDz0,1507 +nvidia_curand_cu12-10.3.5.147.dist-info/RECORD,, +nvidia_curand_cu12-10.3.5.147.dist-info/WHEEL,sha256=XDTs3wIbcE-BcRO08VJlZpA6z9OaC1mOKPCGGGwuM2g,109 +nvidia_curand_cu12-10.3.5.147.dist-info/top_level.txt,sha256=fTkAtiFuL16nUrB9ytDDtpytz2t0B4NvYTnRzwAhO14,7 diff --git a/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/WHEEL b/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..e6c30e957cfb045017a9fef3430bb8ee87c4a074 --- /dev/null +++ b/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.42.0) +Root-Is-Purelib: true +Tag: py3-none-manylinux2014_x86_64 + diff --git a/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/top_level.txt b/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..862f7abf232cdfbb928609856247292e81c9decb --- /dev/null +++ b/lib/python3.1/site-packages/nvidia_curand_cu12-10.3.5.147.dist-info/top_level.txt @@ -0,0 +1 @@ +nvidia diff --git a/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/INSTALLER b/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/License.txt b/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/License.txt new file mode 100644 index 0000000000000000000000000000000000000000..b491c70e0aef319022ded661e111ddbd45b8a17f --- /dev/null +++ b/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/License.txt @@ -0,0 +1,1568 @@ +End User License Agreement +-------------------------- + + +Preface +------- + +The Software License Agreement in Chapter 1 and the Supplement +in Chapter 2 contain license terms and conditions that govern +the use of NVIDIA software. By accepting this agreement, you +agree to comply with all the terms and conditions applicable +to the product(s) included herein. + + +NVIDIA Driver + + +Description + +This package contains the operating system driver and +fundamental system software components for NVIDIA GPUs. + + +NVIDIA CUDA Toolkit + + +Description + +The NVIDIA CUDA Toolkit provides command-line and graphical +tools for building, debugging and optimizing the performance +of applications accelerated by NVIDIA GPUs, runtime and math +libraries, and documentation including programming guides, +user manuals, and API references. + + +Default Install Location of CUDA Toolkit + +Windows platform: + +%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v#.# + +Linux platform: + +/usr/local/cuda-#.# + +Mac platform: + +/Developer/NVIDIA/CUDA-#.# + + +NVIDIA CUDA Samples + + +Description + +This package includes over 100+ CUDA examples that demonstrate +various CUDA programming principles, and efficient CUDA +implementation of algorithms in specific application domains. + + +Default Install Location of CUDA Samples + +Windows platform: + +%ProgramData%\NVIDIA Corporation\CUDA Samples\v#.# + +Linux platform: + +/usr/local/cuda-#.#/samples + +and + +$HOME/NVIDIA_CUDA-#.#_Samples + +Mac platform: + +/Developer/NVIDIA/CUDA-#.#/samples + + +NVIDIA Nsight Visual Studio Edition (Windows only) + + +Description + +NVIDIA Nsight Development Platform, Visual Studio Edition is a +development environment integrated into Microsoft Visual +Studio that provides tools for debugging, profiling, analyzing +and optimizing your GPU computing and graphics applications. + + +Default Install Location of Nsight Visual Studio Edition + +Windows platform: + +%ProgramFiles(x86)%\NVIDIA Corporation\Nsight Visual Studio Edition #.# + + +1. License Agreement for NVIDIA Software Development Kits +--------------------------------------------------------- + + +Release Date: July 26, 2018 +--------------------------- + + +Important NoticeRead before downloading, installing, +copying or using the licensed software: +------------------------------------------------------- + +This license agreement, including exhibits attached +("Agreement”) is a legal agreement between you and NVIDIA +Corporation ("NVIDIA") and governs your use of a NVIDIA +software development kit (“SDK”). + +Each SDK has its own set of software and materials, but here +is a description of the types of items that may be included in +a SDK: source code, header files, APIs, data sets and assets +(examples include images, textures, models, scenes, videos, +native API input/output files), binary software, sample code, +libraries, utility programs, programming code and +documentation. + +This Agreement can be accepted only by an adult of legal age +of majority in the country in which the SDK is used. + +If you are entering into this Agreement on behalf of a company +or other legal entity, you represent that you have the legal +authority to bind the entity to this Agreement, in which case +“you” will mean the entity you represent. + +If you don’t have the required age or authority to accept +this Agreement, or if you don’t accept all the terms and +conditions of this Agreement, do not download, install or use +the SDK. + +You agree to use the SDK only for purposes that are permitted +by (a) this Agreement, and (b) any applicable law, regulation +or generally accepted practices or guidelines in the relevant +jurisdictions. + + +1.1. License + + +1.1.1. License Grant + +Subject to the terms of this Agreement, NVIDIA hereby grants +you a non-exclusive, non-transferable license, without the +right to sublicense (except as expressly provided in this +Agreement) to: + + 1. Install and use the SDK, + + 2. Modify and create derivative works of sample source code + delivered in the SDK, and + + 3. Distribute those portions of the SDK that are identified + in this Agreement as distributable, as incorporated in + object code format into a software application that meets + the distribution requirements indicated in this Agreement. + + +1.1.2. Distribution Requirements + +These are the distribution requirements for you to exercise +the distribution grant: + + 1. Your application must have material additional + functionality, beyond the included portions of the SDK. + + 2. The distributable portions of the SDK shall only be + accessed by your application. + + 3. The following notice shall be included in modifications + and derivative works of sample source code distributed: + “This software contains source code provided by NVIDIA + Corporation.” + + 4. Unless a developer tool is identified in this Agreement + as distributable, it is delivered for your internal use + only. + + 5. The terms under which you distribute your application + must be consistent with the terms of this Agreement, + including (without limitation) terms relating to the + license grant and license restrictions and protection of + NVIDIA’s intellectual property rights. Additionally, you + agree that you will protect the privacy, security and + legal rights of your application users. + + 6. You agree to notify NVIDIA in writing of any known or + suspected distribution or use of the SDK not in compliance + with the requirements of this Agreement, and to enforce + the terms of your agreements with respect to distributed + SDK. + + +1.1.3. Authorized Users + +You may allow employees and contractors of your entity or of +your subsidiary(ies) to access and use the SDK from your +secure network to perform work on your behalf. + +If you are an academic institution you may allow users +enrolled or employed by the academic institution to access and +use the SDK from your secure network. + +You are responsible for the compliance with the terms of this +Agreement by your authorized users. If you become aware that +your authorized users didn’t follow the terms of this +Agreement, you agree to take reasonable steps to resolve the +non-compliance and prevent new occurrences. + + +1.1.4. Pre-Release SDK + +The SDK versions identified as alpha, beta, preview or +otherwise as pre-release, may not be fully functional, may +contain errors or design flaws, and may have reduced or +different security, privacy, accessibility, availability, and +reliability standards relative to commercial versions of +NVIDIA software and materials. Use of a pre-release SDK may +result in unexpected results, loss of data, project delays or +other unpredictable damage or loss. + +You may use a pre-release SDK at your own risk, understanding +that pre-release SDKs are not intended for use in production +or business-critical systems. + +NVIDIA may choose not to make available a commercial version +of any pre-release SDK. NVIDIA may also choose to abandon +development and terminate the availability of a pre-release +SDK at any time without liability. + + +1.1.5. Updates + +NVIDIA may, at its option, make available patches, workarounds +or other updates to this SDK. Unless the updates are provided +with their separate governing terms, they are deemed part of +the SDK licensed to you as provided in this Agreement. You +agree that the form and content of the SDK that NVIDIA +provides may change without prior notice to you. While NVIDIA +generally maintains compatibility between versions, NVIDIA may +in some cases make changes that introduce incompatibilities in +future versions of the SDK. + + +1.1.6. Third Party Licenses + +The SDK may come bundled with, or otherwise include or be +distributed with, third party software licensed by a NVIDIA +supplier and/or open source software provided under an open +source license. Use of third party software is subject to the +third-party license terms, or in the absence of third party +terms, the terms of this Agreement. Copyright to third party +software is held by the copyright holders indicated in the +third-party software or license. + + +1.1.7. Reservation of Rights + +NVIDIA reserves all rights, title, and interest in and to the +SDK, not expressly granted to you under this Agreement. + + +1.2. Limitations + +The following license limitations apply to your use of the +SDK: + + 1. You may not reverse engineer, decompile or disassemble, + or remove copyright or other proprietary notices from any + portion of the SDK or copies of the SDK. + + 2. Except as expressly provided in this Agreement, you may + not copy, sell, rent, sublicense, transfer, distribute, + modify, or create derivative works of any portion of the + SDK. For clarity, you may not distribute or sublicense the + SDK as a stand-alone product. + + 3. Unless you have an agreement with NVIDIA for this + purpose, you may not indicate that an application created + with the SDK is sponsored or endorsed by NVIDIA. + + 4. You may not bypass, disable, or circumvent any + encryption, security, digital rights management or + authentication mechanism in the SDK. + + 5. You may not use the SDK in any manner that would cause it + to become subject to an open source software license. As + examples, licenses that require as a condition of use, + modification, and/or distribution that the SDK be: + + a. Disclosed or distributed in source code form; + + b. Licensed for the purpose of making derivative works; + or + + c. Redistributable at no charge. + + 6. Unless you have an agreement with NVIDIA for this + purpose, you may not use the SDK with any system or + application where the use or failure of the system or + application can reasonably be expected to threaten or + result in personal injury, death, or catastrophic loss. + Examples include use in avionics, navigation, military, + medical, life support or other life critical applications. + NVIDIA does not design, test or manufacture the SDK for + these critical uses and NVIDIA shall not be liable to you + or any third party, in whole or in part, for any claims or + damages arising from such uses. + + 7. You agree to defend, indemnify and hold harmless NVIDIA + and its affiliates, and their respective employees, + contractors, agents, officers and directors, from and + against any and all claims, damages, obligations, losses, + liabilities, costs or debt, fines, restitutions and + expenses (including but not limited to attorney’s fees + and costs incident to establishing the right of + indemnification) arising out of or related to your use of + the SDK outside of the scope of this Agreement, or not in + compliance with its terms. + + +1.3. Ownership + + 1. NVIDIA or its licensors hold all rights, title and + interest in and to the SDK and its modifications and + derivative works, including their respective intellectual + property rights, subject to your rights described in this + section. This SDK may include software and materials from + NVIDIA’s licensors, and these licensors are intended + third party beneficiaries that may enforce this Agreement + with respect to their intellectual property rights. + + 2. You hold all rights, title and interest in and to your + applications and your derivative works of the sample + source code delivered in the SDK, including their + respective intellectual property rights, subject to + NVIDIA’s rights described in this section. + + 3. You may, but don’t have to, provide to NVIDIA + suggestions, feature requests or other feedback regarding + the SDK, including possible enhancements or modifications + to the SDK. For any feedback that you voluntarily provide, + you hereby grant NVIDIA and its affiliates a perpetual, + non-exclusive, worldwide, irrevocable license to use, + reproduce, modify, license, sublicense (through multiple + tiers of sublicensees), and distribute (through multiple + tiers of distributors) it without the payment of any + royalties or fees to you. NVIDIA will use feedback at its + choice. NVIDIA is constantly looking for ways to improve + its products, so you may send feedback to NVIDIA through + the developer portal at https://developer.nvidia.com. + + +1.4. No Warranties + +THE SDK IS PROVIDED BY NVIDIA “AS IS” AND “WITH ALL +FAULTS.” TO THE MAXIMUM EXTENT PERMITTED BY LAW, NVIDIA AND +ITS AFFILIATES EXPRESSLY DISCLAIM ALL WARRANTIES OF ANY KIND +OR NATURE, WHETHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING, +BUT NOT LIMITED TO, ANY WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE, TITLE, NON-INFRINGEMENT, OR THE +ABSENCE OF ANY DEFECTS THEREIN, WHETHER LATENT OR PATENT. NO +WARRANTY IS MADE ON THE BASIS OF TRADE USAGE, COURSE OF +DEALING OR COURSE OF TRADE. + + +1.5. Limitation of Liability + +TO THE MAXIMUM EXTENT PERMITTED BY LAW, NVIDIA AND ITS +AFFILIATES SHALL NOT BE LIABLE FOR ANY SPECIAL, INCIDENTAL, +PUNITIVE OR CONSEQUENTIAL DAMAGES, OR ANY LOST PROFITS, LOSS +OF USE, LOSS OF DATA OR LOSS OF GOODWILL, OR THE COSTS OF +PROCURING SUBSTITUTE PRODUCTS, ARISING OUT OF OR IN CONNECTION +WITH THIS AGREEMENT OR THE USE OR PERFORMANCE OF THE SDK, +WHETHER SUCH LIABILITY ARISES FROM ANY CLAIM BASED UPON BREACH +OF CONTRACT, BREACH OF WARRANTY, TORT (INCLUDING NEGLIGENCE), +PRODUCT LIABILITY OR ANY OTHER CAUSE OF ACTION OR THEORY OF +LIABILITY. IN NO EVENT WILL NVIDIA’S AND ITS AFFILIATES +TOTAL CUMULATIVE LIABILITY UNDER OR ARISING OUT OF THIS +AGREEMENT EXCEED US$10.00. THE NATURE OF THE LIABILITY OR THE +NUMBER OF CLAIMS OR SUITS SHALL NOT ENLARGE OR EXTEND THIS +LIMIT. + +These exclusions and limitations of liability shall apply +regardless if NVIDIA or its affiliates have been advised of +the possibility of such damages, and regardless of whether a +remedy fails its essential purpose. These exclusions and +limitations of liability form an essential basis of the +bargain between the parties, and, absent any of these +exclusions or limitations of liability, the provisions of this +Agreement, including, without limitation, the economic terms, +would be substantially different. + + +1.6. Termination + + 1. This Agreement will continue to apply until terminated by + either you or NVIDIA as described below. + + 2. If you want to terminate this Agreement, you may do so by + stopping to use the SDK. + + 3. NVIDIA may, at any time, terminate this Agreement if: + + a. (i) you fail to comply with any term of this + Agreement and the non-compliance is not fixed within + thirty (30) days following notice from NVIDIA (or + immediately if you violate NVIDIA’s intellectual + property rights); + + b. (ii) you commence or participate in any legal + proceeding against NVIDIA with respect to the SDK; or + + c. (iii) NVIDIA decides to no longer provide the SDK in + a country or, in NVIDIA’s sole discretion, the + continued use of it is no longer commercially viable. + + 4. Upon any termination of this Agreement, you agree to + promptly discontinue use of the SDK and destroy all copies + in your possession or control. Your prior distributions in + accordance with this Agreement are not affected by the + termination of this Agreement. Upon written request, you + will certify in writing that you have complied with your + commitments under this section. Upon any termination of + this Agreement all provisions survive except for the + license grant provisions. + + +1.7. General + +If you wish to assign this Agreement or your rights and +obligations, including by merger, consolidation, dissolution +or operation of law, contact NVIDIA to ask for permission. Any +attempted assignment not approved by NVIDIA in writing shall +be void and of no effect. NVIDIA may assign, delegate or +transfer this Agreement and its rights and obligations, and if +to a non-affiliate you will be notified. + +You agree to cooperate with NVIDIA and provide reasonably +requested information to verify your compliance with this +Agreement. + +This Agreement will be governed in all respects by the laws of +the United States and of the State of Delaware as those laws +are applied to contracts entered into and performed entirely +within Delaware by Delaware residents, without regard to the +conflicts of laws principles. The United Nations Convention on +Contracts for the International Sale of Goods is specifically +disclaimed. You agree to all terms of this Agreement in the +English language. + +The state or federal courts residing in Santa Clara County, +California shall have exclusive jurisdiction over any dispute +or claim arising out of this Agreement. Notwithstanding this, +you agree that NVIDIA shall still be allowed to apply for +injunctive remedies or an equivalent type of urgent legal +relief in any jurisdiction. + +If any court of competent jurisdiction determines that any +provision of this Agreement is illegal, invalid or +unenforceable, such provision will be construed as limited to +the extent necessary to be consistent with and fully +enforceable under the law and the remaining provisions will +remain in full force and effect. Unless otherwise specified, +remedies are cumulative. + +Each party acknowledges and agrees that the other is an +independent contractor in the performance of this Agreement. + +The SDK has been developed entirely at private expense and is +“commercial items” consisting of “commercial computer +software” and “commercial computer software +documentation” provided with RESTRICTED RIGHTS. Use, +duplication or disclosure by the U.S. Government or a U.S. +Government subcontractor is subject to the restrictions in +this Agreement pursuant to DFARS 227.7202-3(a) or as set forth +in subparagraphs (c)(1) and (2) of the Commercial Computer +Software - Restricted Rights clause at FAR 52.227-19, as +applicable. Contractor/manufacturer is NVIDIA, 2788 San Tomas +Expressway, Santa Clara, CA 95051. + +The SDK is subject to United States export laws and +regulations. You agree that you will not ship, transfer or +export the SDK into any country, or use the SDK in any manner, +prohibited by the United States Bureau of Industry and +Security or economic sanctions regulations administered by the +U.S. Department of Treasury’s Office of Foreign Assets +Control (OFAC), or any applicable export laws, restrictions or +regulations. These laws include restrictions on destinations, +end users and end use. By accepting this Agreement, you +confirm that you are not a resident or citizen of any country +currently embargoed by the U.S. and that you are not otherwise +prohibited from receiving the SDK. + +Any notice delivered by NVIDIA to you under this Agreement +will be delivered via mail, email or fax. You agree that any +notices that NVIDIA sends you electronically will satisfy any +legal communication requirements. Please direct your legal +notices or other correspondence to NVIDIA Corporation, 2788 +San Tomas Expressway, Santa Clara, California 95051, United +States of America, Attention: Legal Department. + +This Agreement and any exhibits incorporated into this +Agreement constitute the entire agreement of the parties with +respect to the subject matter of this Agreement and supersede +all prior negotiations or documentation exchanged between the +parties relating to this SDK license. Any additional and/or +conflicting terms on documents issued by you are null, void, +and invalid. Any amendment or waiver under this Agreement +shall be in writing and signed by representatives of both +parties. + + +2. CUDA Toolkit Supplement to Software License Agreement for +NVIDIA Software Development Kits +------------------------------------------------------------ + + +Release date: August 16, 2018 +----------------------------- + +The terms in this supplement govern your use of the NVIDIA +CUDA Toolkit SDK under the terms of your license agreement +(“Agreement”) as modified by this supplement. Capitalized +terms used but not defined below have the meaning assigned to +them in the Agreement. + +This supplement is an exhibit to the Agreement and is +incorporated as an integral part of the Agreement. In the +event of conflict between the terms in this supplement and the +terms in the Agreement, the terms in this supplement govern. + + +2.1. License Scope + +The SDK is licensed for you to develop applications only for +use in systems with NVIDIA GPUs. + + +2.2. Distribution + +The portions of the SDK that are distributable under the +Agreement are listed in Attachment A. + + +2.3. Operating Systems + +Those portions of the SDK designed exclusively for use on the +Linux or FreeBSD operating systems, or other operating systems +derived from the source code to these operating systems, may +be copied and redistributed for use in accordance with this +Agreement, provided that the object code files are not +modified in any way (except for unzipping of compressed +files). + + +2.4. Audio and Video Encoders and Decoders + +You acknowledge and agree that it is your sole responsibility +to obtain any additional third-party licenses required to +make, have made, use, have used, sell, import, and offer for +sale your products or services that include or incorporate any +third-party software and content relating to audio and/or +video encoders and decoders from, including but not limited +to, Microsoft, Thomson, Fraunhofer IIS, Sisvel S.p.A., +MPEG-LA, and Coding Technologies. NVIDIA does not grant to you +under this Agreement any necessary patent or other rights with +respect to any audio and/or video encoders and decoders. + + +2.5. Licensing + +If the distribution terms in this Agreement are not suitable +for your organization, or for any questions regarding this +Agreement, please contact NVIDIA at +nvidia-compute-license-questions@nvidia.com. + + +2.6. Attachment A + +The following portions of the SDK are distributable under the +Agreement: + +Component + +CUDA Runtime + +Windows + +cudart.dll, cudart_static.lib, cudadevrt.lib + +Mac OSX + +libcudart.dylib, libcudart_static.a, libcudadevrt.a + +Linux + +libcudart.so, libcudart_static.a, libcudadevrt.a + +Android + +libcudart.so, libcudart_static.a, libcudadevrt.a + +Component + +CUDA FFT Library + +Windows + +cufft.dll, cufftw.dll, cufft.lib, cufftw.lib + +Mac OSX + +libcufft.dylib, libcufft_static.a, libcufftw.dylib, +libcufftw_static.a + +Linux + +libcufft.so, libcufft_static.a, libcufftw.so, +libcufftw_static.a + +Android + +libcufft.so, libcufft_static.a, libcufftw.so, +libcufftw_static.a + +Component + +CUDA BLAS Library + +Windows + +cublas.dll, cublasLt.dll + +Mac OSX + +libcublas.dylib, libcublasLt.dylib, libcublas_static.a, +libcublasLt_static.a + +Linux + +libcublas.so, libcublasLt.so, libcublas_static.a, +libcublasLt_static.a + +Android + +libcublas.so, libcublasLt.so, libcublas_static.a, +libcublasLt_static.a + +Component + +NVIDIA "Drop-in" BLAS Library + +Windows + +nvblas.dll + +Mac OSX + +libnvblas.dylib + +Linux + +libnvblas.so + +Component + +CUDA Sparse Matrix Library + +Windows + +cusparse.dll, cusparse.lib + +Mac OSX + +libcusparse.dylib, libcusparse_static.a + +Linux + +libcusparse.so, libcusparse_static.a + +Android + +libcusparse.so, libcusparse_static.a + +Component + +CUDA Linear Solver Library + +Windows + +cusolver.dll, cusolver.lib + +Mac OSX + +libcusolver.dylib, libcusolver_static.a + +Linux + +libcusolver.so, libcusolver_static.a + +Android + +libcusolver.so, libcusolver_static.a + +Component + +CUDA Random Number Generation Library + +Windows + +curand.dll, curand.lib + +Mac OSX + +libcurand.dylib, libcurand_static.a + +Linux + +libcurand.so, libcurand_static.a + +Android + +libcurand.so, libcurand_static.a + +Component + +CUDA Accelerated Graph Library + +Component + +NVIDIA Performance Primitives Library + +Windows + +nppc.dll, nppc.lib, nppial.dll, nppial.lib, nppicc.dll, +nppicc.lib, nppicom.dll, nppicom.lib, nppidei.dll, +nppidei.lib, nppif.dll, nppif.lib, nppig.dll, nppig.lib, +nppim.dll, nppim.lib, nppist.dll, nppist.lib, nppisu.dll, +nppisu.lib, nppitc.dll, nppitc.lib, npps.dll, npps.lib + +Mac OSX + +libnppc.dylib, libnppc_static.a, libnppial.dylib, +libnppial_static.a, libnppicc.dylib, libnppicc_static.a, +libnppicom.dylib, libnppicom_static.a, libnppidei.dylib, +libnppidei_static.a, libnppif.dylib, libnppif_static.a, +libnppig.dylib, libnppig_static.a, libnppim.dylib, +libnppisu_static.a, libnppitc.dylib, libnppitc_static.a, +libnpps.dylib, libnpps_static.a + +Linux + +libnppc.so, libnppc_static.a, libnppial.so, +libnppial_static.a, libnppicc.so, libnppicc_static.a, +libnppicom.so, libnppicom_static.a, libnppidei.so, +libnppidei_static.a, libnppif.so, libnppif_static.a +libnppig.so, libnppig_static.a, libnppim.so, +libnppim_static.a, libnppist.so, libnppist_static.a, +libnppisu.so, libnppisu_static.a, libnppitc.so +libnppitc_static.a, libnpps.so, libnpps_static.a + +Android + +libnppc.so, libnppc_static.a, libnppial.so, +libnppial_static.a, libnppicc.so, libnppicc_static.a, +libnppicom.so, libnppicom_static.a, libnppidei.so, +libnppidei_static.a, libnppif.so, libnppif_static.a +libnppig.so, libnppig_static.a, libnppim.so, +libnppim_static.a, libnppist.so, libnppist_static.a, +libnppisu.so, libnppisu_static.a, libnppitc.so +libnppitc_static.a, libnpps.so, libnpps_static.a + +Component + +NVIDIA JPEG Library + +Linux + +libnvjpeg.so, libnvjpeg_static.a + +Component + +Internal common library required for statically linking to +cuBLAS, cuSPARSE, cuFFT, cuRAND, nvJPEG and NPP + +Mac OSX + +libculibos.a + +Linux + +libculibos.a + +Component + +NVIDIA Runtime Compilation Library and Header + +All + +nvrtc.h + +Windows + +nvrtc.dll, nvrtc-builtins.dll + +Mac OSX + +libnvrtc.dylib, libnvrtc-builtins.dylib + +Linux + +libnvrtc.so, libnvrtc-builtins.so + +Component + +NVIDIA Optimizing Compiler Library + +Windows + +nvvm.dll + +Mac OSX + +libnvvm.dylib + +Linux + +libnvvm.so + +Component + +NVIDIA Common Device Math Functions Library + +Windows + +libdevice.10.bc + +Mac OSX + +libdevice.10.bc + +Linux + +libdevice.10.bc + +Component + +CUDA Occupancy Calculation Header Library + +All + +cuda_occupancy.h + +Component + +CUDA Half Precision Headers + +All + +cuda_fp16.h, cuda_fp16.hpp + +Component + +CUDA Profiling Tools Interface (CUPTI) Library + +Windows + +cupti.dll + +Mac OSX + +libcupti.dylib + +Linux + +libcupti.so + +Component + +NVIDIA Tools Extension Library + +Windows + +nvToolsExt.dll, nvToolsExt.lib + +Mac OSX + +libnvToolsExt.dylib + +Linux + +libnvToolsExt.so + +Component + +NVIDIA CUDA Driver Libraries + +Linux + +libcuda.so, libnvidia-fatbinaryloader.so, +libnvidia-ptxjitcompiler.so + +The NVIDIA CUDA Driver Libraries are only distributable in +applications that meet this criteria: + + 1. The application was developed starting from a NVIDIA CUDA + container obtained from Docker Hub or the NVIDIA GPU + Cloud, and + + 2. The resulting application is packaged as a Docker + container and distributed to users on Docker Hub or the + NVIDIA GPU Cloud only. + + +2.7. Attachment B + + +Additional Licensing Obligations + +The following third party components included in the SOFTWARE +are licensed to Licensee pursuant to the following terms and +conditions: + + 1. Licensee's use of the GDB third party component is + subject to the terms and conditions of GNU GPL v3: + + This product includes copyrighted third-party software licensed + under the terms of the GNU General Public License v3 ("GPL v3"). + All third-party software packages are copyright by their respective + authors. GPL v3 terms and conditions are hereby incorporated into + the Agreement by this reference: http://www.gnu.org/licenses/gpl.txt + + Consistent with these licensing requirements, the software + listed below is provided under the terms of the specified + open source software licenses. To obtain source code for + software provided under licenses that require + redistribution of source code, including the GNU General + Public License (GPL) and GNU Lesser General Public License + (LGPL), contact oss-requests@nvidia.com. This offer is + valid for a period of three (3) years from the date of the + distribution of this product by NVIDIA CORPORATION. + + Component License + CUDA-GDB GPL v3 + + 2. Licensee represents and warrants that any and all third + party licensing and/or royalty payment obligations in + connection with Licensee's use of the H.264 video codecs + are solely the responsibility of Licensee. + + 3. Licensee's use of the Thrust library is subject to the + terms and conditions of the Apache License Version 2.0. + All third-party software packages are copyright by their + respective authors. Apache License Version 2.0 terms and + conditions are hereby incorporated into the Agreement by + this reference. + http://www.apache.org/licenses/LICENSE-2.0.html + + In addition, Licensee acknowledges the following notice: + Thrust includes source code from the Boost Iterator, + Tuple, System, and Random Number libraries. + + Boost Software License - Version 1.0 - August 17th, 2003 + . . . . + + Permission is hereby granted, free of charge, to any person or + organization obtaining a copy of the software and accompanying + documentation covered by this license (the "Software") to use, + reproduce, display, distribute, execute, and transmit the Software, + and to prepare derivative works of the Software, and to permit + third-parties to whom the Software is furnished to do so, all + subject to the following: + + The copyright notices in the Software and this entire statement, + including the above license grant, this restriction and the following + disclaimer, must be included in all copies of the Software, in whole + or in part, and all derivative works of the Software, unless such + copies or derivative works are solely in the form of machine-executable + object code generated by a source language processor. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE AND + NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR + ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR ANY DAMAGES OR + OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + 4. Licensee's use of the LLVM third party component is + subject to the following terms and conditions: + + ====================================================== + LLVM Release License + ====================================================== + University of Illinois/NCSA + Open Source License + + Copyright (c) 2003-2010 University of Illinois at Urbana-Champaign. + All rights reserved. + + Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to + deal with the Software without restriction, including without limitation the + rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at Urbana- + Champaign, nor the names of its contributors may be used to endorse or + promote products derived from this Software without specific prior + written permission. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS WITH THE SOFTWARE. + + 5. Licensee's use (e.g. nvprof) of the PCRE third party + component is subject to the following terms and + conditions: + + ------------ + PCRE LICENCE + ------------ + PCRE is a library of functions to support regular expressions whose syntax + and semantics are as close as possible to those of the Perl 5 language. + Release 8 of PCRE is distributed under the terms of the "BSD" licence, as + specified below. The documentation for PCRE, supplied in the "doc" + directory, is distributed under the same terms as the software itself. The + basic library functions are written in C and are freestanding. Also + included in the distribution is a set of C++ wrapper functions, and a just- + in-time compiler that can be used to optimize pattern matching. These are + both optional features that can be omitted when the library is built. + + THE BASIC LIBRARY FUNCTIONS + --------------------------- + Written by: Philip Hazel + Email local part: ph10 + Email domain: cam.ac.uk + University of Cambridge Computing Service, + Cambridge, England. + Copyright (c) 1997-2012 University of Cambridge + All rights reserved. + + PCRE JUST-IN-TIME COMPILATION SUPPORT + ------------------------------------- + Written by: Zoltan Herczeg + Email local part: hzmester + Emain domain: freemail.hu + Copyright(c) 2010-2012 Zoltan Herczeg + All rights reserved. + + STACK-LESS JUST-IN-TIME COMPILER + -------------------------------- + Written by: Zoltan Herczeg + Email local part: hzmester + Emain domain: freemail.hu + Copyright(c) 2009-2012 Zoltan Herczeg + All rights reserved. + + THE C++ WRAPPER FUNCTIONS + ------------------------- + Contributed by: Google Inc. + Copyright (c) 2007-2012, Google Inc. + All rights reserved. + + THE "BSD" LICENCE + ----------------- + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + 6. Some of the cuBLAS library routines were written by or + derived from code written by Vasily Volkov and are subject + to the Modified Berkeley Software Distribution License as + follows: + + Copyright (c) 2007-2009, Regents of the University of California + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of the University of California, Berkeley nor + the names of its contributors may be used to endorse or promote + products derived from this software without specific prior + written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + 7. Some of the cuBLAS library routines were written by or + derived from code written by Davide Barbieri and are + subject to the Modified Berkeley Software Distribution + License as follows: + + Copyright (c) 2008-2009 Davide Barbieri @ University of Rome Tor Vergata. + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * The name of the author may not be used to endorse or promote + products derived from this software without specific prior + written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + 8. Some of the cuBLAS library routines were derived from + code developed by the University of Tennessee and are + subject to the Modified Berkeley Software Distribution + License as follows: + + Copyright (c) 2010 The University of Tennessee. + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer listed in this license in the documentation and/or + other materials provided with the distribution. + * Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 9. Some of the cuBLAS library routines were written by or + derived from code written by Jonathan Hogg and are subject + to the Modified Berkeley Software Distribution License as + follows: + + Copyright (c) 2012, The Science and Technology Facilities Council (STFC). + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of the STFC nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE STFC BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 10. Some of the cuBLAS library routines were written by or + derived from code written by Ahmad M. Abdelfattah, David + Keyes, and Hatem Ltaief, and are subject to the Apache + License, Version 2.0, as follows: + + -- (C) Copyright 2013 King Abdullah University of Science and Technology + Authors: + Ahmad Abdelfattah (ahmad.ahmad@kaust.edu.sa) + David Keyes (david.keyes@kaust.edu.sa) + Hatem Ltaief (hatem.ltaief@kaust.edu.sa) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the King Abdullah University of Science and + Technology nor the names of its contributors may be used to endorse + or promote products derived from this software without specific prior + written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE + + 11. Some of the cuSPARSE library routines were written by or + derived from code written by Li-Wen Chang and are subject + to the NCSA Open Source License as follows: + + Copyright (c) 2012, University of Illinois. + + All rights reserved. + + Developed by: IMPACT Group, University of Illinois, http://impact.crhc.illinois.edu + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal with the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimers in the documentation and/or other materials provided + with the distribution. + * Neither the names of IMPACT Group, University of Illinois, nor + the names of its contributors may be used to endorse or promote + products derived from this Software without specific prior + written permission. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE + SOFTWARE. + + 12. Some of the cuRAND library routines were written by or + derived from code written by Mutsuo Saito and Makoto + Matsumoto and are subject to the following license: + + Copyright (c) 2009, 2010 Mutsuo Saito, Makoto Matsumoto and Hiroshima + University. All rights reserved. + + Copyright (c) 2011 Mutsuo Saito, Makoto Matsumoto, Hiroshima + University and University of Tokyo. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of the Hiroshima University nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 13. Some of the cuRAND library routines were derived from + code developed by D. E. Shaw Research and are subject to + the following license: + + Copyright 2010-2011, D. E. Shaw Research. + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions, and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions, and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of D. E. Shaw Research nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 14. Some of the Math library routines were written by or + derived from code developed by Norbert Juffa and are + subject to the following license: + + Copyright (c) 2015-2017, Norbert Juffa + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 15. Licensee's use of the lz4 third party component is + subject to the following terms and conditions: + + Copyright (C) 2011-2013, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 16. The NPP library uses code from the Boost Math Toolkit, + and is subject to the following license: + + Boost Software License - Version 1.0 - August 17th, 2003 + . . . . + + Permission is hereby granted, free of charge, to any person or + organization obtaining a copy of the software and accompanying + documentation covered by this license (the "Software") to use, + reproduce, display, distribute, execute, and transmit the Software, + and to prepare derivative works of the Software, and to permit + third-parties to whom the Software is furnished to do so, all + subject to the following: + + The copyright notices in the Software and this entire statement, + including the above license grant, this restriction and the following + disclaimer, must be included in all copies of the Software, in whole + or in part, and all derivative works of the Software, unless such + copies or derivative works are solely in the form of machine-executable + object code generated by a source language processor. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE AND + NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR + ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR ANY DAMAGES OR + OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + 17. Portions of the Nsight Eclipse Edition is subject to the + following license: + + The Eclipse Foundation makes available all content in this plug-in + ("Content"). Unless otherwise indicated below, the Content is provided + to you under the terms and conditions of the Eclipse Public License + Version 1.0 ("EPL"). A copy of the EPL is available at http:// + www.eclipse.org/legal/epl-v10.html. For purposes of the EPL, "Program" + will mean the Content. + + If you did not receive this Content directly from the Eclipse + Foundation, the Content is being redistributed by another party + ("Redistributor") and different terms and conditions may apply to your + use of any object code in the Content. Check the Redistributor's + license that was provided with the Content. If no such license exists, + contact the Redistributor. Unless otherwise indicated below, the terms + and conditions of the EPL still apply to any source code in the + Content and such source code may be obtained at http://www.eclipse.org. + + 18. Some of the cuBLAS library routines uses code from + OpenAI, which is subject to the following license: + + License URL + https://github.com/openai/openai-gemm/blob/master/LICENSE + + License Text + The MIT License + + Copyright (c) 2016 OpenAI (http://openai.com), 2016 Google Inc. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + + 19. Licensee's use of the Visual Studio Setup Configuration + Samples is subject to the following license: + + The MIT License (MIT) + Copyright (C) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + 20. Licensee's use of linmath.h header for CPU functions for + GL vector/matrix operations from lunarG is subject to the + Apache License Version 2.0. + + 21. The DX12-CUDA sample uses the d3dx12.h header, which is + subject to the MIT license . + +----------------- diff --git a/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/METADATA b/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..e8ef8db3f0102373f6e6897f513e5a80fb4e4dbd --- /dev/null +++ b/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/METADATA @@ -0,0 +1,36 @@ +Metadata-Version: 2.1 +Name: nvidia-cusparse-cu12 +Version: 12.3.1.170 +Summary: CUSPARSE native runtime libraries +Home-page: https://developer.nvidia.com/cuda-zone +Author: Nvidia CUDA Installer Team +Author-email: cuda_installer@nvidia.com +License: NVIDIA Proprietary Software +Keywords: cuda,nvidia,runtime,machine learning,deep learning +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Education +Classifier: Intended Audience :: Science/Research +Classifier: License :: Other/Proprietary License +Classifier: Natural Language :: English +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Topic :: Scientific/Engineering +Classifier: Topic :: Scientific/Engineering :: Mathematics +Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence +Classifier: Topic :: Software Development +Classifier: Topic :: Software Development :: Libraries +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX :: Linux +Requires-Python: >=3 +License-File: License.txt +Requires-Dist: nvidia-nvjitlink-cu12 + +CUSPARSE native runtime libraries diff --git a/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/RECORD b/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..d0f16dc961297b326cc5491595cfe1db32e94ddc --- /dev/null +++ b/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/RECORD @@ -0,0 +1,17 @@ +nvidia/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +nvidia/__pycache__/__init__.cpython-310.pyc,, +nvidia/cusparse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +nvidia/cusparse/__pycache__/__init__.cpython-310.pyc,, +nvidia/cusparse/include/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +nvidia/cusparse/include/__pycache__/__init__.cpython-310.pyc,, +nvidia/cusparse/include/cusparse.h,sha256=g-jyaV5nBdnCExOVQOd7lf4lMBoR3KvriiNHUoXNgXw,295187 +nvidia/cusparse/include/cusparse_v2.h,sha256=jkH2A9hYc-TEF0vuQ_SurbhPNEHkYGUIRuxKXhFAqnw,2587 +nvidia/cusparse/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +nvidia/cusparse/lib/__pycache__/__init__.cpython-310.pyc,, +nvidia/cusparse/lib/libcusparse.so.12,sha256=y0KIyWVFOoAgZTwP2SzfzRK7WmRoQ9Zj6dTl9SD4kVw,281313984 +nvidia_cusparse_cu12-12.3.1.170.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +nvidia_cusparse_cu12-12.3.1.170.dist-info/License.txt,sha256=rW9YU_ugyg0VnQ9Y1JrkmDDC-Mk_epJki5zpCttMbM0,59262 +nvidia_cusparse_cu12-12.3.1.170.dist-info/METADATA,sha256=ivWTVzZv8-4bxy3u1Ce2wd3qUZeGJ3jTuH3-8u7AILM,1550 +nvidia_cusparse_cu12-12.3.1.170.dist-info/RECORD,, +nvidia_cusparse_cu12-12.3.1.170.dist-info/WHEEL,sha256=XDTs3wIbcE-BcRO08VJlZpA6z9OaC1mOKPCGGGwuM2g,109 +nvidia_cusparse_cu12-12.3.1.170.dist-info/top_level.txt,sha256=fTkAtiFuL16nUrB9ytDDtpytz2t0B4NvYTnRzwAhO14,7 diff --git a/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/WHEEL b/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..e6c30e957cfb045017a9fef3430bb8ee87c4a074 --- /dev/null +++ b/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.42.0) +Root-Is-Purelib: true +Tag: py3-none-manylinux2014_x86_64 + diff --git a/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/top_level.txt b/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..862f7abf232cdfbb928609856247292e81c9decb --- /dev/null +++ b/lib/python3.1/site-packages/nvidia_cusparse_cu12-12.3.1.170.dist-info/top_level.txt @@ -0,0 +1 @@ +nvidia diff --git a/lib/python3.1/site-packages/orjson-3.11.2.dist-info/INSTALLER b/lib/python3.1/site-packages/orjson-3.11.2.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.1/site-packages/orjson-3.11.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.1/site-packages/orjson-3.11.2.dist-info/METADATA b/lib/python3.1/site-packages/orjson-3.11.2.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..5df4e17c58d2ac0863be1f0f61a507253c7a2af5 --- /dev/null +++ b/lib/python3.1/site-packages/orjson-3.11.2.dist-info/METADATA @@ -0,0 +1,27 @@ +Metadata-Version: 2.4 +Name: orjson +Version: 3.11.2 +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: MacOS +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX :: Linux +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python +Classifier: Programming Language :: Rust +Classifier: Typing :: Typed +License-File: LICENSE-APACHE +License-File: LICENSE-MIT +Requires-Python: >=3.9 +Project-URL: source, https://github.com/ijl/orjson +Project-URL: documentation, https://github.com/ijl/orjson +Project-URL: changelog, https://github.com/ijl/orjson/blob/master/CHANGELOG.md diff --git a/lib/python3.1/site-packages/orjson-3.11.2.dist-info/RECORD b/lib/python3.1/site-packages/orjson-3.11.2.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..c37f5d42c71d7144bebbd96712d28f57abbd4b27 --- /dev/null +++ b/lib/python3.1/site-packages/orjson-3.11.2.dist-info/RECORD @@ -0,0 +1,11 @@ +orjson-3.11.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +orjson-3.11.2.dist-info/METADATA,sha256=jYSjsY93Sa8j2d9qBc-fN5zpAmywYUfZk964Y48xkrw,1206 +orjson-3.11.2.dist-info/RECORD,, +orjson-3.11.2.dist-info/WHEEL,sha256=eVFlkE1vBrKFbMiva8QY5dljMhlrWHHdD9pKbf8maq4,129 +orjson-3.11.2.dist-info/licenses/LICENSE-APACHE,sha256=pg7qgXUUUxZo1-AHZXMUSf4U0FnTJJ4LyTs23kX3WfI,10847 +orjson-3.11.2.dist-info/licenses/LICENSE-MIT,sha256=I_GOA9xJ35FiL-KnYXZJdATkbO2KcV2dK2enRGVxzKM,1023 +orjson/__init__.py,sha256=1SdrKCoU_OHDXXY54LqX9Q-12MjiMWic3r52Cp_CYXA,589 +orjson/__init__.pyi,sha256=eGvdxmKvRiEkhsV9VuM4naUMwrJjLVOKiTFxMgQxgAI,728 +orjson/__pycache__/__init__.cpython-310.pyc,, +orjson/orjson.cpython-310-x86_64-linux-gnu.so,sha256=r9QoYEVCrA-xtU0E6vltz0QZRgr6LwY4Uco6WOFxkX8,241360 +orjson/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/lib/python3.1/site-packages/orjson-3.11.2.dist-info/WHEEL b/lib/python3.1/site-packages/orjson-3.11.2.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..b3fa2801e450c965844b53f64f731d31628f8cd0 --- /dev/null +++ b/lib/python3.1/site-packages/orjson-3.11.2.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: maturin (1.9.3) +Root-Is-Purelib: false +Tag: cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64 diff --git a/lib/python3.1/site-packages/orjson-3.11.2.dist-info/licenses/LICENSE-APACHE b/lib/python3.1/site-packages/orjson-3.11.2.dist-info/licenses/LICENSE-APACHE new file mode 100644 index 0000000000000000000000000000000000000000..16fe87b06e802f094b3fbb0894b137bca2b16ef1 --- /dev/null +++ b/lib/python3.1/site-packages/orjson-3.11.2.dist-info/licenses/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/lib/python3.1/site-packages/orjson-3.11.2.dist-info/licenses/LICENSE-MIT b/lib/python3.1/site-packages/orjson-3.11.2.dist-info/licenses/LICENSE-MIT new file mode 100644 index 0000000000000000000000000000000000000000..31aa79387f27e730e33d871925e152e35e428031 --- /dev/null +++ b/lib/python3.1/site-packages/orjson-3.11.2.dist-info/licenses/LICENSE-MIT @@ -0,0 +1,23 @@ +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/lib/python3.1/site-packages/orjson/__init__.py b/lib/python3.1/site-packages/orjson/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..892fd682bcdab2a194e880655276b8a6c15c02a4 --- /dev/null +++ b/lib/python3.1/site-packages/orjson/__init__.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from .orjson import * +from .orjson import __version__ + +__all__ = ( + "__version__", + "dumps", + "Fragment", + "JSONDecodeError", + "JSONEncodeError", + "loads", + "OPT_APPEND_NEWLINE", + "OPT_INDENT_2", + "OPT_NAIVE_UTC", + "OPT_NON_STR_KEYS", + "OPT_OMIT_MICROSECONDS", + "OPT_PASSTHROUGH_DATACLASS", + "OPT_PASSTHROUGH_DATETIME", + "OPT_PASSTHROUGH_SUBCLASS", + "OPT_SERIALIZE_DATACLASS", + "OPT_SERIALIZE_NUMPY", + "OPT_SERIALIZE_UUID", + "OPT_SORT_KEYS", + "OPT_STRICT_INTEGER", + "OPT_UTC_Z", +) diff --git a/lib/python3.1/site-packages/orjson/__init__.pyi b/lib/python3.1/site-packages/orjson/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..4efc910fbc8defade5002ea5729ae1839dc11cca --- /dev/null +++ b/lib/python3.1/site-packages/orjson/__init__.pyi @@ -0,0 +1,32 @@ +import json +from typing import Any, Callable + +__version__: str + +def dumps( + __obj: Any, + default: Callable[[Any], Any] | None = ..., + option: int | None = ..., +) -> bytes: ... +def loads(__obj: bytes | bytearray | memoryview | str) -> Any: ... + +class JSONDecodeError(json.JSONDecodeError): ... +class JSONEncodeError(TypeError): ... + +class Fragment(tuple): + contents: bytes | str + +OPT_APPEND_NEWLINE: int +OPT_INDENT_2: int +OPT_NAIVE_UTC: int +OPT_NON_STR_KEYS: int +OPT_OMIT_MICROSECONDS: int +OPT_PASSTHROUGH_DATACLASS: int +OPT_PASSTHROUGH_DATETIME: int +OPT_PASSTHROUGH_SUBCLASS: int +OPT_SERIALIZE_DATACLASS: int +OPT_SERIALIZE_NUMPY: int +OPT_SERIALIZE_UUID: int +OPT_SORT_KEYS: int +OPT_STRICT_INTEGER: int +OPT_UTC_Z: int diff --git a/lib/python3.1/site-packages/orjson/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/site-packages/orjson/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d5a9786e82a147db6b022d3f418181d5d1c20d5b Binary files /dev/null and b/lib/python3.1/site-packages/orjson/__pycache__/__init__.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/orjson/py.typed b/lib/python3.1/site-packages/orjson/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.1/site-packages/packaging/__init__.py b/lib/python3.1/site-packages/packaging/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d45c22cfd88e2b88976b46d4171a502b602658ec --- /dev/null +++ b/lib/python3.1/site-packages/packaging/__init__.py @@ -0,0 +1,15 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +__title__ = "packaging" +__summary__ = "Core utilities for Python packages" +__uri__ = "https://github.com/pypa/packaging" + +__version__ = "25.0" + +__author__ = "Donald Stufft and individual contributors" +__email__ = "donald@stufft.io" + +__license__ = "BSD-2-Clause or Apache-2.0" +__copyright__ = f"2014 {__author__}" diff --git a/lib/python3.1/site-packages/packaging/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb927f9cfc2b26a54ad183de1955fc15b90ed9ed Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/__init__.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/__pycache__/_elffile.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/_elffile.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ac3c89ec2d94fbd6bc40fb2cff256176725b9eed Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/_elffile.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/__pycache__/_manylinux.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/_manylinux.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..779cc1b23dcbe87795507c690b38e89bbad3122c Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/_manylinux.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/__pycache__/_musllinux.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/_musllinux.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..623e9ca0052e9c585f646901d19af1243b5e1b30 Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/_musllinux.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/__pycache__/_parser.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/_parser.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef532ca72c9794401b757c81b5b758f7eda9856c Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/_parser.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/__pycache__/_structures.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/_structures.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b9313706f76867811f0231d672dbc127330f8426 Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/_structures.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/__pycache__/_tokenizer.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/_tokenizer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fa14db6bbb5abb6c7501a95ec58fcb7b0979f1bc Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/_tokenizer.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/__pycache__/markers.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/markers.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9e2d09c08b32d66e49722d7e31e97784c906703c Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/markers.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/__pycache__/metadata.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/metadata.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..36025a56d25fe955c891a3d89300152adddfc158 Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/metadata.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/__pycache__/requirements.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/requirements.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66129d2f8d04210cdcff3d79c45a382ced3ea8f6 Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/requirements.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/__pycache__/specifiers.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/specifiers.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b41ed30348b019172139c006ff9da49a5cf23be5 Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/specifiers.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/__pycache__/tags.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/tags.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f04e3ed4eb7be8cdaf559aa106aec2ce90a012e Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/tags.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/__pycache__/utils.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..533153f392c970e1dc174cd8f105feee5290eaa9 Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/utils.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/__pycache__/version.cpython-310.pyc b/lib/python3.1/site-packages/packaging/__pycache__/version.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7262c515481942e62c41b8cea177e9a2d4565c8f Binary files /dev/null and b/lib/python3.1/site-packages/packaging/__pycache__/version.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/_elffile.py b/lib/python3.1/site-packages/packaging/_elffile.py new file mode 100644 index 0000000000000000000000000000000000000000..7a5afc33b0a2401cf509bb6a8d1143d5230324d9 --- /dev/null +++ b/lib/python3.1/site-packages/packaging/_elffile.py @@ -0,0 +1,109 @@ +""" +ELF file parser. + +This provides a class ``ELFFile`` that parses an ELF executable in a similar +interface to ``ZipFile``. Only the read interface is implemented. + +Based on: https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca +ELF header: https://refspecs.linuxfoundation.org/elf/gabi4+/ch4.eheader.html +""" + +from __future__ import annotations + +import enum +import os +import struct +from typing import IO + + +class ELFInvalid(ValueError): + pass + + +class EIClass(enum.IntEnum): + C32 = 1 + C64 = 2 + + +class EIData(enum.IntEnum): + Lsb = 1 + Msb = 2 + + +class EMachine(enum.IntEnum): + I386 = 3 + S390 = 22 + Arm = 40 + X8664 = 62 + AArc64 = 183 + + +class ELFFile: + """ + Representation of an ELF executable. + """ + + def __init__(self, f: IO[bytes]) -> None: + self._f = f + + try: + ident = self._read("16B") + except struct.error as e: + raise ELFInvalid("unable to parse identification") from e + magic = bytes(ident[:4]) + if magic != b"\x7fELF": + raise ELFInvalid(f"invalid magic: {magic!r}") + + self.capacity = ident[4] # Format for program header (bitness). + self.encoding = ident[5] # Data structure encoding (endianness). + + try: + # e_fmt: Format for program header. + # p_fmt: Format for section header. + # p_idx: Indexes to find p_type, p_offset, and p_filesz. + e_fmt, self._p_fmt, self._p_idx = { + (1, 1): ("HHIIIIIHHH", ">IIIIIIII", (0, 1, 4)), # 32-bit MSB. + (2, 1): ("HHIQQQIHHH", ">IIQQQQQQ", (0, 2, 5)), # 64-bit MSB. + }[(self.capacity, self.encoding)] + except KeyError as e: + raise ELFInvalid( + f"unrecognized capacity ({self.capacity}) or encoding ({self.encoding})" + ) from e + + try: + ( + _, + self.machine, # Architecture type. + _, + _, + self._e_phoff, # Offset of program header. + _, + self.flags, # Processor-specific flags. + _, + self._e_phentsize, # Size of section. + self._e_phnum, # Number of sections. + ) = self._read(e_fmt) + except struct.error as e: + raise ELFInvalid("unable to parse machine and section information") from e + + def _read(self, fmt: str) -> tuple[int, ...]: + return struct.unpack(fmt, self._f.read(struct.calcsize(fmt))) + + @property + def interpreter(self) -> str | None: + """ + The path recorded in the ``PT_INTERP`` section header. + """ + for index in range(self._e_phnum): + self._f.seek(self._e_phoff + self._e_phentsize * index) + try: + data = self._read(self._p_fmt) + except struct.error: + continue + if data[self._p_idx[0]] != 3: # Not PT_INTERP. + continue + self._f.seek(data[self._p_idx[1]]) + return os.fsdecode(self._f.read(data[self._p_idx[2]])).strip("\0") + return None diff --git a/lib/python3.1/site-packages/packaging/_manylinux.py b/lib/python3.1/site-packages/packaging/_manylinux.py new file mode 100644 index 0000000000000000000000000000000000000000..95f55762e86f08af76aaa95f5cd147d4517c6d66 --- /dev/null +++ b/lib/python3.1/site-packages/packaging/_manylinux.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import collections +import contextlib +import functools +import os +import re +import sys +import warnings +from typing import Generator, Iterator, NamedTuple, Sequence + +from ._elffile import EIClass, EIData, ELFFile, EMachine + +EF_ARM_ABIMASK = 0xFF000000 +EF_ARM_ABI_VER5 = 0x05000000 +EF_ARM_ABI_FLOAT_HARD = 0x00000400 + + +# `os.PathLike` not a generic type until Python 3.9, so sticking with `str` +# as the type for `path` until then. +@contextlib.contextmanager +def _parse_elf(path: str) -> Generator[ELFFile | None, None, None]: + try: + with open(path, "rb") as f: + yield ELFFile(f) + except (OSError, TypeError, ValueError): + yield None + + +def _is_linux_armhf(executable: str) -> bool: + # hard-float ABI can be detected from the ELF header of the running + # process + # https://static.docs.arm.com/ihi0044/g/aaelf32.pdf + with _parse_elf(executable) as f: + return ( + f is not None + and f.capacity == EIClass.C32 + and f.encoding == EIData.Lsb + and f.machine == EMachine.Arm + and f.flags & EF_ARM_ABIMASK == EF_ARM_ABI_VER5 + and f.flags & EF_ARM_ABI_FLOAT_HARD == EF_ARM_ABI_FLOAT_HARD + ) + + +def _is_linux_i686(executable: str) -> bool: + with _parse_elf(executable) as f: + return ( + f is not None + and f.capacity == EIClass.C32 + and f.encoding == EIData.Lsb + and f.machine == EMachine.I386 + ) + + +def _have_compatible_abi(executable: str, archs: Sequence[str]) -> bool: + if "armv7l" in archs: + return _is_linux_armhf(executable) + if "i686" in archs: + return _is_linux_i686(executable) + allowed_archs = { + "x86_64", + "aarch64", + "ppc64", + "ppc64le", + "s390x", + "loongarch64", + "riscv64", + } + return any(arch in allowed_archs for arch in archs) + + +# If glibc ever changes its major version, we need to know what the last +# minor version was, so we can build the complete list of all versions. +# For now, guess what the highest minor version might be, assume it will +# be 50 for testing. Once this actually happens, update the dictionary +# with the actual value. +_LAST_GLIBC_MINOR: dict[int, int] = collections.defaultdict(lambda: 50) + + +class _GLibCVersion(NamedTuple): + major: int + minor: int + + +def _glibc_version_string_confstr() -> str | None: + """ + Primary implementation of glibc_version_string using os.confstr. + """ + # os.confstr is quite a bit faster than ctypes.DLL. It's also less likely + # to be broken or missing. This strategy is used in the standard library + # platform module. + # https://github.com/python/cpython/blob/fcf1d003bf4f0100c/Lib/platform.py#L175-L183 + try: + # Should be a string like "glibc 2.17". + version_string: str | None = os.confstr("CS_GNU_LIBC_VERSION") + assert version_string is not None + _, version = version_string.rsplit() + except (AssertionError, AttributeError, OSError, ValueError): + # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... + return None + return version + + +def _glibc_version_string_ctypes() -> str | None: + """ + Fallback implementation of glibc_version_string using ctypes. + """ + try: + import ctypes + except ImportError: + return None + + # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen + # manpage says, "If filename is NULL, then the returned handle is for the + # main program". This way we can let the linker do the work to figure out + # which libc our process is actually using. + # + # We must also handle the special case where the executable is not a + # dynamically linked executable. This can occur when using musl libc, + # for example. In this situation, dlopen() will error, leading to an + # OSError. Interestingly, at least in the case of musl, there is no + # errno set on the OSError. The single string argument used to construct + # OSError comes from libc itself and is therefore not portable to + # hard code here. In any case, failure to call dlopen() means we + # can proceed, so we bail on our attempt. + try: + process_namespace = ctypes.CDLL(None) + except OSError: + return None + + try: + gnu_get_libc_version = process_namespace.gnu_get_libc_version + except AttributeError: + # Symbol doesn't exist -> therefore, we are not linked to + # glibc. + return None + + # Call gnu_get_libc_version, which returns a string like "2.5" + gnu_get_libc_version.restype = ctypes.c_char_p + version_str: str = gnu_get_libc_version() + # py2 / py3 compatibility: + if not isinstance(version_str, str): + version_str = version_str.decode("ascii") + + return version_str + + +def _glibc_version_string() -> str | None: + """Returns glibc version string, or None if not using glibc.""" + return _glibc_version_string_confstr() or _glibc_version_string_ctypes() + + +def _parse_glibc_version(version_str: str) -> tuple[int, int]: + """Parse glibc version. + + We use a regexp instead of str.split because we want to discard any + random junk that might come after the minor version -- this might happen + in patched/forked versions of glibc (e.g. Linaro's version of glibc + uses version strings like "2.20-2014.11"). See gh-3588. + """ + m = re.match(r"(?P[0-9]+)\.(?P[0-9]+)", version_str) + if not m: + warnings.warn( + f"Expected glibc version with 2 components major.minor, got: {version_str}", + RuntimeWarning, + stacklevel=2, + ) + return -1, -1 + return int(m.group("major")), int(m.group("minor")) + + +@functools.lru_cache +def _get_glibc_version() -> tuple[int, int]: + version_str = _glibc_version_string() + if version_str is None: + return (-1, -1) + return _parse_glibc_version(version_str) + + +# From PEP 513, PEP 600 +def _is_compatible(arch: str, version: _GLibCVersion) -> bool: + sys_glibc = _get_glibc_version() + if sys_glibc < version: + return False + # Check for presence of _manylinux module. + try: + import _manylinux + except ImportError: + return True + if hasattr(_manylinux, "manylinux_compatible"): + result = _manylinux.manylinux_compatible(version[0], version[1], arch) + if result is not None: + return bool(result) + return True + if version == _GLibCVersion(2, 5): + if hasattr(_manylinux, "manylinux1_compatible"): + return bool(_manylinux.manylinux1_compatible) + if version == _GLibCVersion(2, 12): + if hasattr(_manylinux, "manylinux2010_compatible"): + return bool(_manylinux.manylinux2010_compatible) + if version == _GLibCVersion(2, 17): + if hasattr(_manylinux, "manylinux2014_compatible"): + return bool(_manylinux.manylinux2014_compatible) + return True + + +_LEGACY_MANYLINUX_MAP = { + # CentOS 7 w/ glibc 2.17 (PEP 599) + (2, 17): "manylinux2014", + # CentOS 6 w/ glibc 2.12 (PEP 571) + (2, 12): "manylinux2010", + # CentOS 5 w/ glibc 2.5 (PEP 513) + (2, 5): "manylinux1", +} + + +def platform_tags(archs: Sequence[str]) -> Iterator[str]: + """Generate manylinux tags compatible to the current platform. + + :param archs: Sequence of compatible architectures. + The first one shall be the closest to the actual architecture and be the part of + platform tag after the ``linux_`` prefix, e.g. ``x86_64``. + The ``linux_`` prefix is assumed as a prerequisite for the current platform to + be manylinux-compatible. + + :returns: An iterator of compatible manylinux tags. + """ + if not _have_compatible_abi(sys.executable, archs): + return + # Oldest glibc to be supported regardless of architecture is (2, 17). + too_old_glibc2 = _GLibCVersion(2, 16) + if set(archs) & {"x86_64", "i686"}: + # On x86/i686 also oldest glibc to be supported is (2, 5). + too_old_glibc2 = _GLibCVersion(2, 4) + current_glibc = _GLibCVersion(*_get_glibc_version()) + glibc_max_list = [current_glibc] + # We can assume compatibility across glibc major versions. + # https://sourceware.org/bugzilla/show_bug.cgi?id=24636 + # + # Build a list of maximum glibc versions so that we can + # output the canonical list of all glibc from current_glibc + # down to too_old_glibc2, including all intermediary versions. + for glibc_major in range(current_glibc.major - 1, 1, -1): + glibc_minor = _LAST_GLIBC_MINOR[glibc_major] + glibc_max_list.append(_GLibCVersion(glibc_major, glibc_minor)) + for arch in archs: + for glibc_max in glibc_max_list: + if glibc_max.major == too_old_glibc2.major: + min_minor = too_old_glibc2.minor + else: + # For other glibc major versions oldest supported is (x, 0). + min_minor = -1 + for glibc_minor in range(glibc_max.minor, min_minor, -1): + glibc_version = _GLibCVersion(glibc_max.major, glibc_minor) + tag = "manylinux_{}_{}".format(*glibc_version) + if _is_compatible(arch, glibc_version): + yield f"{tag}_{arch}" + # Handle the legacy manylinux1, manylinux2010, manylinux2014 tags. + if glibc_version in _LEGACY_MANYLINUX_MAP: + legacy_tag = _LEGACY_MANYLINUX_MAP[glibc_version] + if _is_compatible(arch, glibc_version): + yield f"{legacy_tag}_{arch}" diff --git a/lib/python3.1/site-packages/packaging/_musllinux.py b/lib/python3.1/site-packages/packaging/_musllinux.py new file mode 100644 index 0000000000000000000000000000000000000000..d2bf30b56319ba862c5c9a1a39a87c6d1cb68718 --- /dev/null +++ b/lib/python3.1/site-packages/packaging/_musllinux.py @@ -0,0 +1,85 @@ +"""PEP 656 support. + +This module implements logic to detect if the currently running Python is +linked against musl, and what musl version is used. +""" + +from __future__ import annotations + +import functools +import re +import subprocess +import sys +from typing import Iterator, NamedTuple, Sequence + +from ._elffile import ELFFile + + +class _MuslVersion(NamedTuple): + major: int + minor: int + + +def _parse_musl_version(output: str) -> _MuslVersion | None: + lines = [n for n in (n.strip() for n in output.splitlines()) if n] + if len(lines) < 2 or lines[0][:4] != "musl": + return None + m = re.match(r"Version (\d+)\.(\d+)", lines[1]) + if not m: + return None + return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2))) + + +@functools.lru_cache +def _get_musl_version(executable: str) -> _MuslVersion | None: + """Detect currently-running musl runtime version. + + This is done by checking the specified executable's dynamic linking + information, and invoking the loader to parse its output for a version + string. If the loader is musl, the output would be something like:: + + musl libc (x86_64) + Version 1.2.2 + Dynamic Program Loader + """ + try: + with open(executable, "rb") as f: + ld = ELFFile(f).interpreter + except (OSError, TypeError, ValueError): + return None + if ld is None or "musl" not in ld: + return None + proc = subprocess.run([ld], stderr=subprocess.PIPE, text=True) + return _parse_musl_version(proc.stderr) + + +def platform_tags(archs: Sequence[str]) -> Iterator[str]: + """Generate musllinux tags compatible to the current platform. + + :param archs: Sequence of compatible architectures. + The first one shall be the closest to the actual architecture and be the part of + platform tag after the ``linux_`` prefix, e.g. ``x86_64``. + The ``linux_`` prefix is assumed as a prerequisite for the current platform to + be musllinux-compatible. + + :returns: An iterator of compatible musllinux tags. + """ + sys_musl = _get_musl_version(sys.executable) + if sys_musl is None: # Python not dynamically linked against musl. + return + for arch in archs: + for minor in range(sys_musl.minor, -1, -1): + yield f"musllinux_{sys_musl.major}_{minor}_{arch}" + + +if __name__ == "__main__": # pragma: no cover + import sysconfig + + plat = sysconfig.get_platform() + assert plat.startswith("linux-"), "not linux" + + print("plat:", plat) + print("musl:", _get_musl_version(sys.executable)) + print("tags:", end=" ") + for t in platform_tags(re.sub(r"[.-]", "_", plat.split("-", 1)[-1])): + print(t, end="\n ") diff --git a/lib/python3.1/site-packages/packaging/_parser.py b/lib/python3.1/site-packages/packaging/_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..0007c0aa64aeae24c7e0ca90790a25ba51dbf7d6 --- /dev/null +++ b/lib/python3.1/site-packages/packaging/_parser.py @@ -0,0 +1,353 @@ +"""Handwritten parser of dependency specifiers. + +The docstring for each __parse_* function contains EBNF-inspired grammar representing +the implementation. +""" + +from __future__ import annotations + +import ast +from typing import NamedTuple, Sequence, Tuple, Union + +from ._tokenizer import DEFAULT_RULES, Tokenizer + + +class Node: + def __init__(self, value: str) -> None: + self.value = value + + def __str__(self) -> str: + return self.value + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}('{self}')>" + + def serialize(self) -> str: + raise NotImplementedError + + +class Variable(Node): + def serialize(self) -> str: + return str(self) + + +class Value(Node): + def serialize(self) -> str: + return f'"{self}"' + + +class Op(Node): + def serialize(self) -> str: + return str(self) + + +MarkerVar = Union[Variable, Value] +MarkerItem = Tuple[MarkerVar, Op, MarkerVar] +MarkerAtom = Union[MarkerItem, Sequence["MarkerAtom"]] +MarkerList = Sequence[Union["MarkerList", MarkerAtom, str]] + + +class ParsedRequirement(NamedTuple): + name: str + url: str + extras: list[str] + specifier: str + marker: MarkerList | None + + +# -------------------------------------------------------------------------------------- +# Recursive descent parser for dependency specifier +# -------------------------------------------------------------------------------------- +def parse_requirement(source: str) -> ParsedRequirement: + return _parse_requirement(Tokenizer(source, rules=DEFAULT_RULES)) + + +def _parse_requirement(tokenizer: Tokenizer) -> ParsedRequirement: + """ + requirement = WS? IDENTIFIER WS? extras WS? requirement_details + """ + tokenizer.consume("WS") + + name_token = tokenizer.expect( + "IDENTIFIER", expected="package name at the start of dependency specifier" + ) + name = name_token.text + tokenizer.consume("WS") + + extras = _parse_extras(tokenizer) + tokenizer.consume("WS") + + url, specifier, marker = _parse_requirement_details(tokenizer) + tokenizer.expect("END", expected="end of dependency specifier") + + return ParsedRequirement(name, url, extras, specifier, marker) + + +def _parse_requirement_details( + tokenizer: Tokenizer, +) -> tuple[str, str, MarkerList | None]: + """ + requirement_details = AT URL (WS requirement_marker?)? + | specifier WS? (requirement_marker)? + """ + + specifier = "" + url = "" + marker = None + + if tokenizer.check("AT"): + tokenizer.read() + tokenizer.consume("WS") + + url_start = tokenizer.position + url = tokenizer.expect("URL", expected="URL after @").text + if tokenizer.check("END", peek=True): + return (url, specifier, marker) + + tokenizer.expect("WS", expected="whitespace after URL") + + # The input might end after whitespace. + if tokenizer.check("END", peek=True): + return (url, specifier, marker) + + marker = _parse_requirement_marker( + tokenizer, span_start=url_start, after="URL and whitespace" + ) + else: + specifier_start = tokenizer.position + specifier = _parse_specifier(tokenizer) + tokenizer.consume("WS") + + if tokenizer.check("END", peek=True): + return (url, specifier, marker) + + marker = _parse_requirement_marker( + tokenizer, + span_start=specifier_start, + after=( + "version specifier" + if specifier + else "name and no valid version specifier" + ), + ) + + return (url, specifier, marker) + + +def _parse_requirement_marker( + tokenizer: Tokenizer, *, span_start: int, after: str +) -> MarkerList: + """ + requirement_marker = SEMICOLON marker WS? + """ + + if not tokenizer.check("SEMICOLON"): + tokenizer.raise_syntax_error( + f"Expected end or semicolon (after {after})", + span_start=span_start, + ) + tokenizer.read() + + marker = _parse_marker(tokenizer) + tokenizer.consume("WS") + + return marker + + +def _parse_extras(tokenizer: Tokenizer) -> list[str]: + """ + extras = (LEFT_BRACKET wsp* extras_list? wsp* RIGHT_BRACKET)? + """ + if not tokenizer.check("LEFT_BRACKET", peek=True): + return [] + + with tokenizer.enclosing_tokens( + "LEFT_BRACKET", + "RIGHT_BRACKET", + around="extras", + ): + tokenizer.consume("WS") + extras = _parse_extras_list(tokenizer) + tokenizer.consume("WS") + + return extras + + +def _parse_extras_list(tokenizer: Tokenizer) -> list[str]: + """ + extras_list = identifier (wsp* ',' wsp* identifier)* + """ + extras: list[str] = [] + + if not tokenizer.check("IDENTIFIER"): + return extras + + extras.append(tokenizer.read().text) + + while True: + tokenizer.consume("WS") + if tokenizer.check("IDENTIFIER", peek=True): + tokenizer.raise_syntax_error("Expected comma between extra names") + elif not tokenizer.check("COMMA"): + break + + tokenizer.read() + tokenizer.consume("WS") + + extra_token = tokenizer.expect("IDENTIFIER", expected="extra name after comma") + extras.append(extra_token.text) + + return extras + + +def _parse_specifier(tokenizer: Tokenizer) -> str: + """ + specifier = LEFT_PARENTHESIS WS? version_many WS? RIGHT_PARENTHESIS + | WS? version_many WS? + """ + with tokenizer.enclosing_tokens( + "LEFT_PARENTHESIS", + "RIGHT_PARENTHESIS", + around="version specifier", + ): + tokenizer.consume("WS") + parsed_specifiers = _parse_version_many(tokenizer) + tokenizer.consume("WS") + + return parsed_specifiers + + +def _parse_version_many(tokenizer: Tokenizer) -> str: + """ + version_many = (SPECIFIER (WS? COMMA WS? SPECIFIER)*)? + """ + parsed_specifiers = "" + while tokenizer.check("SPECIFIER"): + span_start = tokenizer.position + parsed_specifiers += tokenizer.read().text + if tokenizer.check("VERSION_PREFIX_TRAIL", peek=True): + tokenizer.raise_syntax_error( + ".* suffix can only be used with `==` or `!=` operators", + span_start=span_start, + span_end=tokenizer.position + 1, + ) + if tokenizer.check("VERSION_LOCAL_LABEL_TRAIL", peek=True): + tokenizer.raise_syntax_error( + "Local version label can only be used with `==` or `!=` operators", + span_start=span_start, + span_end=tokenizer.position, + ) + tokenizer.consume("WS") + if not tokenizer.check("COMMA"): + break + parsed_specifiers += tokenizer.read().text + tokenizer.consume("WS") + + return parsed_specifiers + + +# -------------------------------------------------------------------------------------- +# Recursive descent parser for marker expression +# -------------------------------------------------------------------------------------- +def parse_marker(source: str) -> MarkerList: + return _parse_full_marker(Tokenizer(source, rules=DEFAULT_RULES)) + + +def _parse_full_marker(tokenizer: Tokenizer) -> MarkerList: + retval = _parse_marker(tokenizer) + tokenizer.expect("END", expected="end of marker expression") + return retval + + +def _parse_marker(tokenizer: Tokenizer) -> MarkerList: + """ + marker = marker_atom (BOOLOP marker_atom)+ + """ + expression = [_parse_marker_atom(tokenizer)] + while tokenizer.check("BOOLOP"): + token = tokenizer.read() + expr_right = _parse_marker_atom(tokenizer) + expression.extend((token.text, expr_right)) + return expression + + +def _parse_marker_atom(tokenizer: Tokenizer) -> MarkerAtom: + """ + marker_atom = WS? LEFT_PARENTHESIS WS? marker WS? RIGHT_PARENTHESIS WS? + | WS? marker_item WS? + """ + + tokenizer.consume("WS") + if tokenizer.check("LEFT_PARENTHESIS", peek=True): + with tokenizer.enclosing_tokens( + "LEFT_PARENTHESIS", + "RIGHT_PARENTHESIS", + around="marker expression", + ): + tokenizer.consume("WS") + marker: MarkerAtom = _parse_marker(tokenizer) + tokenizer.consume("WS") + else: + marker = _parse_marker_item(tokenizer) + tokenizer.consume("WS") + return marker + + +def _parse_marker_item(tokenizer: Tokenizer) -> MarkerItem: + """ + marker_item = WS? marker_var WS? marker_op WS? marker_var WS? + """ + tokenizer.consume("WS") + marker_var_left = _parse_marker_var(tokenizer) + tokenizer.consume("WS") + marker_op = _parse_marker_op(tokenizer) + tokenizer.consume("WS") + marker_var_right = _parse_marker_var(tokenizer) + tokenizer.consume("WS") + return (marker_var_left, marker_op, marker_var_right) + + +def _parse_marker_var(tokenizer: Tokenizer) -> MarkerVar: + """ + marker_var = VARIABLE | QUOTED_STRING + """ + if tokenizer.check("VARIABLE"): + return process_env_var(tokenizer.read().text.replace(".", "_")) + elif tokenizer.check("QUOTED_STRING"): + return process_python_str(tokenizer.read().text) + else: + tokenizer.raise_syntax_error( + message="Expected a marker variable or quoted string" + ) + + +def process_env_var(env_var: str) -> Variable: + if env_var in ("platform_python_implementation", "python_implementation"): + return Variable("platform_python_implementation") + else: + return Variable(env_var) + + +def process_python_str(python_str: str) -> Value: + value = ast.literal_eval(python_str) + return Value(str(value)) + + +def _parse_marker_op(tokenizer: Tokenizer) -> Op: + """ + marker_op = IN | NOT IN | OP + """ + if tokenizer.check("IN"): + tokenizer.read() + return Op("in") + elif tokenizer.check("NOT"): + tokenizer.read() + tokenizer.expect("WS", expected="whitespace after 'not'") + tokenizer.expect("IN", expected="'in' after 'not'") + return Op("not in") + elif tokenizer.check("OP"): + return Op(tokenizer.read().text) + else: + return tokenizer.raise_syntax_error( + "Expected marker operator, one of <=, <, !=, ==, >=, >, ~=, ===, in, not in" + ) diff --git a/lib/python3.1/site-packages/packaging/_structures.py b/lib/python3.1/site-packages/packaging/_structures.py new file mode 100644 index 0000000000000000000000000000000000000000..90a6465f9682c886363eea5327dac64bf623a6ff --- /dev/null +++ b/lib/python3.1/site-packages/packaging/_structures.py @@ -0,0 +1,61 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + + +class InfinityType: + def __repr__(self) -> str: + return "Infinity" + + def __hash__(self) -> int: + return hash(repr(self)) + + def __lt__(self, other: object) -> bool: + return False + + def __le__(self, other: object) -> bool: + return False + + def __eq__(self, other: object) -> bool: + return isinstance(other, self.__class__) + + def __gt__(self, other: object) -> bool: + return True + + def __ge__(self, other: object) -> bool: + return True + + def __neg__(self: object) -> "NegativeInfinityType": + return NegativeInfinity + + +Infinity = InfinityType() + + +class NegativeInfinityType: + def __repr__(self) -> str: + return "-Infinity" + + def __hash__(self) -> int: + return hash(repr(self)) + + def __lt__(self, other: object) -> bool: + return True + + def __le__(self, other: object) -> bool: + return True + + def __eq__(self, other: object) -> bool: + return isinstance(other, self.__class__) + + def __gt__(self, other: object) -> bool: + return False + + def __ge__(self, other: object) -> bool: + return False + + def __neg__(self: object) -> InfinityType: + return Infinity + + +NegativeInfinity = NegativeInfinityType() diff --git a/lib/python3.1/site-packages/packaging/_tokenizer.py b/lib/python3.1/site-packages/packaging/_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..d28a9b6cf5daca9fa3ce5a8c8ae583a57ee84f12 --- /dev/null +++ b/lib/python3.1/site-packages/packaging/_tokenizer.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import contextlib +import re +from dataclasses import dataclass +from typing import Iterator, NoReturn + +from .specifiers import Specifier + + +@dataclass +class Token: + name: str + text: str + position: int + + +class ParserSyntaxError(Exception): + """The provided source text could not be parsed correctly.""" + + def __init__( + self, + message: str, + *, + source: str, + span: tuple[int, int], + ) -> None: + self.span = span + self.message = message + self.source = source + + super().__init__() + + def __str__(self) -> str: + marker = " " * self.span[0] + "~" * (self.span[1] - self.span[0]) + "^" + return "\n ".join([self.message, self.source, marker]) + + +DEFAULT_RULES: dict[str, str | re.Pattern[str]] = { + "LEFT_PARENTHESIS": r"\(", + "RIGHT_PARENTHESIS": r"\)", + "LEFT_BRACKET": r"\[", + "RIGHT_BRACKET": r"\]", + "SEMICOLON": r";", + "COMMA": r",", + "QUOTED_STRING": re.compile( + r""" + ( + ('[^']*') + | + ("[^"]*") + ) + """, + re.VERBOSE, + ), + "OP": r"(===|==|~=|!=|<=|>=|<|>)", + "BOOLOP": r"\b(or|and)\b", + "IN": r"\bin\b", + "NOT": r"\bnot\b", + "VARIABLE": re.compile( + r""" + \b( + python_version + |python_full_version + |os[._]name + |sys[._]platform + |platform_(release|system) + |platform[._](version|machine|python_implementation) + |python_implementation + |implementation_(name|version) + |extras? + |dependency_groups + )\b + """, + re.VERBOSE, + ), + "SPECIFIER": re.compile( + Specifier._operator_regex_str + Specifier._version_regex_str, + re.VERBOSE | re.IGNORECASE, + ), + "AT": r"\@", + "URL": r"[^ \t]+", + "IDENTIFIER": r"\b[a-zA-Z0-9][a-zA-Z0-9._-]*\b", + "VERSION_PREFIX_TRAIL": r"\.\*", + "VERSION_LOCAL_LABEL_TRAIL": r"\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*", + "WS": r"[ \t]+", + "END": r"$", +} + + +class Tokenizer: + """Context-sensitive token parsing. + + Provides methods to examine the input stream to check whether the next token + matches. + """ + + def __init__( + self, + source: str, + *, + rules: dict[str, str | re.Pattern[str]], + ) -> None: + self.source = source + self.rules: dict[str, re.Pattern[str]] = { + name: re.compile(pattern) for name, pattern in rules.items() + } + self.next_token: Token | None = None + self.position = 0 + + def consume(self, name: str) -> None: + """Move beyond provided token name, if at current position.""" + if self.check(name): + self.read() + + def check(self, name: str, *, peek: bool = False) -> bool: + """Check whether the next token has the provided name. + + By default, if the check succeeds, the token *must* be read before + another check. If `peek` is set to `True`, the token is not loaded and + would need to be checked again. + """ + assert self.next_token is None, ( + f"Cannot check for {name!r}, already have {self.next_token!r}" + ) + assert name in self.rules, f"Unknown token name: {name!r}" + + expression = self.rules[name] + + match = expression.match(self.source, self.position) + if match is None: + return False + if not peek: + self.next_token = Token(name, match[0], self.position) + return True + + def expect(self, name: str, *, expected: str) -> Token: + """Expect a certain token name next, failing with a syntax error otherwise. + + The token is *not* read. + """ + if not self.check(name): + raise self.raise_syntax_error(f"Expected {expected}") + return self.read() + + def read(self) -> Token: + """Consume the next token and return it.""" + token = self.next_token + assert token is not None + + self.position += len(token.text) + self.next_token = None + + return token + + def raise_syntax_error( + self, + message: str, + *, + span_start: int | None = None, + span_end: int | None = None, + ) -> NoReturn: + """Raise ParserSyntaxError at the given position.""" + span = ( + self.position if span_start is None else span_start, + self.position if span_end is None else span_end, + ) + raise ParserSyntaxError( + message, + source=self.source, + span=span, + ) + + @contextlib.contextmanager + def enclosing_tokens( + self, open_token: str, close_token: str, *, around: str + ) -> Iterator[None]: + if self.check(open_token): + open_position = self.position + self.read() + else: + open_position = None + + yield + + if open_position is None: + return + + if not self.check(close_token): + self.raise_syntax_error( + f"Expected matching {close_token} for {open_token}, after {around}", + span_start=open_position, + ) + + self.read() diff --git a/lib/python3.1/site-packages/packaging/licenses/__init__.py b/lib/python3.1/site-packages/packaging/licenses/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6f7f9e6289dabe5bbc45cc8f01ac2449da890d02 --- /dev/null +++ b/lib/python3.1/site-packages/packaging/licenses/__init__.py @@ -0,0 +1,145 @@ +####################################################################################### +# +# Adapted from: +# https://github.com/pypa/hatch/blob/5352e44/backend/src/hatchling/licenses/parse.py +# +# MIT License +# +# Copyright (c) 2017-present Ofek Lev +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this +# software and associated documentation files (the "Software"), to deal in the Software +# without restriction, including without limitation the rights to use, copy, modify, +# merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be included in all copies +# or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# +# With additional allowance of arbitrary `LicenseRef-` identifiers, not just +# `LicenseRef-Public-Domain` and `LicenseRef-Proprietary`. +# +####################################################################################### +from __future__ import annotations + +import re +from typing import NewType, cast + +from packaging.licenses._spdx import EXCEPTIONS, LICENSES + +__all__ = [ + "InvalidLicenseExpression", + "NormalizedLicenseExpression", + "canonicalize_license_expression", +] + +license_ref_allowed = re.compile("^[A-Za-z0-9.-]*$") + +NormalizedLicenseExpression = NewType("NormalizedLicenseExpression", str) + + +class InvalidLicenseExpression(ValueError): + """Raised when a license-expression string is invalid + + >>> canonicalize_license_expression("invalid") + Traceback (most recent call last): + ... + packaging.licenses.InvalidLicenseExpression: Invalid license expression: 'invalid' + """ + + +def canonicalize_license_expression( + raw_license_expression: str, +) -> NormalizedLicenseExpression: + if not raw_license_expression: + message = f"Invalid license expression: {raw_license_expression!r}" + raise InvalidLicenseExpression(message) + + # Pad any parentheses so tokenization can be achieved by merely splitting on + # whitespace. + license_expression = raw_license_expression.replace("(", " ( ").replace(")", " ) ") + licenseref_prefix = "LicenseRef-" + license_refs = { + ref.lower(): "LicenseRef-" + ref[len(licenseref_prefix) :] + for ref in license_expression.split() + if ref.lower().startswith(licenseref_prefix.lower()) + } + + # Normalize to lower case so we can look up licenses/exceptions + # and so boolean operators are Python-compatible. + license_expression = license_expression.lower() + + tokens = license_expression.split() + + # Rather than implementing boolean logic, we create an expression that Python can + # parse. Everything that is not involved with the grammar itself is treated as + # `False` and the expression should evaluate as such. + python_tokens = [] + for token in tokens: + if token not in {"or", "and", "with", "(", ")"}: + python_tokens.append("False") + elif token == "with": + python_tokens.append("or") + elif token == "(" and python_tokens and python_tokens[-1] not in {"or", "and"}: + message = f"Invalid license expression: {raw_license_expression!r}" + raise InvalidLicenseExpression(message) + else: + python_tokens.append(token) + + python_expression = " ".join(python_tokens) + try: + invalid = eval(python_expression, globals(), locals()) + except Exception: + invalid = True + + if invalid is not False: + message = f"Invalid license expression: {raw_license_expression!r}" + raise InvalidLicenseExpression(message) from None + + # Take a final pass to check for unknown licenses/exceptions. + normalized_tokens = [] + for token in tokens: + if token in {"or", "and", "with", "(", ")"}: + normalized_tokens.append(token.upper()) + continue + + if normalized_tokens and normalized_tokens[-1] == "WITH": + if token not in EXCEPTIONS: + message = f"Unknown license exception: {token!r}" + raise InvalidLicenseExpression(message) + + normalized_tokens.append(EXCEPTIONS[token]["id"]) + else: + if token.endswith("+"): + final_token = token[:-1] + suffix = "+" + else: + final_token = token + suffix = "" + + if final_token.startswith("licenseref-"): + if not license_ref_allowed.match(final_token): + message = f"Invalid licenseref: {final_token!r}" + raise InvalidLicenseExpression(message) + normalized_tokens.append(license_refs[final_token] + suffix) + else: + if final_token not in LICENSES: + message = f"Unknown license: {final_token!r}" + raise InvalidLicenseExpression(message) + normalized_tokens.append(LICENSES[final_token]["id"] + suffix) + + normalized_expression = " ".join(normalized_tokens) + + return cast( + NormalizedLicenseExpression, + normalized_expression.replace("( ", "(").replace(" )", ")"), + ) diff --git a/lib/python3.1/site-packages/packaging/licenses/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/site-packages/packaging/licenses/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..826ba3e140610943a6ebed763a2c7b862bcbb649 Binary files /dev/null and b/lib/python3.1/site-packages/packaging/licenses/__pycache__/__init__.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/licenses/__pycache__/_spdx.cpython-310.pyc b/lib/python3.1/site-packages/packaging/licenses/__pycache__/_spdx.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..670dac0d12bf830cbebd8aeea59c0fb359a57925 Binary files /dev/null and b/lib/python3.1/site-packages/packaging/licenses/__pycache__/_spdx.cpython-310.pyc differ diff --git a/lib/python3.1/site-packages/packaging/licenses/_spdx.py b/lib/python3.1/site-packages/packaging/licenses/_spdx.py new file mode 100644 index 0000000000000000000000000000000000000000..eac22276a34ccd73fc9d70c67ca318a49eb11e77 --- /dev/null +++ b/lib/python3.1/site-packages/packaging/licenses/_spdx.py @@ -0,0 +1,759 @@ + +from __future__ import annotations + +from typing import TypedDict + +class SPDXLicense(TypedDict): + id: str + deprecated: bool + +class SPDXException(TypedDict): + id: str + deprecated: bool + + +VERSION = '3.25.0' + +LICENSES: dict[str, SPDXLicense] = { + '0bsd': {'id': '0BSD', 'deprecated': False}, + '3d-slicer-1.0': {'id': '3D-Slicer-1.0', 'deprecated': False}, + 'aal': {'id': 'AAL', 'deprecated': False}, + 'abstyles': {'id': 'Abstyles', 'deprecated': False}, + 'adacore-doc': {'id': 'AdaCore-doc', 'deprecated': False}, + 'adobe-2006': {'id': 'Adobe-2006', 'deprecated': False}, + 'adobe-display-postscript': {'id': 'Adobe-Display-PostScript', 'deprecated': False}, + 'adobe-glyph': {'id': 'Adobe-Glyph', 'deprecated': False}, + 'adobe-utopia': {'id': 'Adobe-Utopia', 'deprecated': False}, + 'adsl': {'id': 'ADSL', 'deprecated': False}, + 'afl-1.1': {'id': 'AFL-1.1', 'deprecated': False}, + 'afl-1.2': {'id': 'AFL-1.2', 'deprecated': False}, + 'afl-2.0': {'id': 'AFL-2.0', 'deprecated': False}, + 'afl-2.1': {'id': 'AFL-2.1', 'deprecated': False}, + 'afl-3.0': {'id': 'AFL-3.0', 'deprecated': False}, + 'afmparse': {'id': 'Afmparse', 'deprecated': False}, + 'agpl-1.0': {'id': 'AGPL-1.0', 'deprecated': True}, + 'agpl-1.0-only': {'id': 'AGPL-1.0-only', 'deprecated': False}, + 'agpl-1.0-or-later': {'id': 'AGPL-1.0-or-later', 'deprecated': False}, + 'agpl-3.0': {'id': 'AGPL-3.0', 'deprecated': True}, + 'agpl-3.0-only': {'id': 'AGPL-3.0-only', 'deprecated': False}, + 'agpl-3.0-or-later': {'id': 'AGPL-3.0-or-later', 'deprecated': False}, + 'aladdin': {'id': 'Aladdin', 'deprecated': False}, + 'amd-newlib': {'id': 'AMD-newlib', 'deprecated': False}, + 'amdplpa': {'id': 'AMDPLPA', 'deprecated': False}, + 'aml': {'id': 'AML', 'deprecated': False}, + 'aml-glslang': {'id': 'AML-glslang', 'deprecated': False}, + 'ampas': {'id': 'AMPAS', 'deprecated': False}, + 'antlr-pd': {'id': 'ANTLR-PD', 'deprecated': False}, + 'antlr-pd-fallback': {'id': 'ANTLR-PD-fallback', 'deprecated': False}, + 'any-osi': {'id': 'any-OSI', 'deprecated': False}, + 'apache-1.0': {'id': 'Apache-1.0', 'deprecated': False}, + 'apache-1.1': {'id': 'Apache-1.1', 'deprecated': False}, + 'apache-2.0': {'id': 'Apache-2.0', 'deprecated': False}, + 'apafml': {'id': 'APAFML', 'deprecated': False}, + 'apl-1.0': {'id': 'APL-1.0', 'deprecated': False}, + 'app-s2p': {'id': 'App-s2p', 'deprecated': False}, + 'apsl-1.0': {'id': 'APSL-1.0', 'deprecated': False}, + 'apsl-1.1': {'id': 'APSL-1.1', 'deprecated': False}, + 'apsl-1.2': {'id': 'APSL-1.2', 'deprecated': False}, + 'apsl-2.0': {'id': 'APSL-2.0', 'deprecated': False}, + 'arphic-1999': {'id': 'Arphic-1999', 'deprecated': False}, + 'artistic-1.0': {'id': 'Artistic-1.0', 'deprecated': False}, + 'artistic-1.0-cl8': {'id': 'Artistic-1.0-cl8', 'deprecated': False}, + 'artistic-1.0-perl': {'id': 'Artistic-1.0-Perl', 'deprecated': False}, + 'artistic-2.0': {'id': 'Artistic-2.0', 'deprecated': False}, + 'aswf-digital-assets-1.0': {'id': 'ASWF-Digital-Assets-1.0', 'deprecated': False}, + 'aswf-digital-assets-1.1': {'id': 'ASWF-Digital-Assets-1.1', 'deprecated': False}, + 'baekmuk': {'id': 'Baekmuk', 'deprecated': False}, + 'bahyph': {'id': 'Bahyph', 'deprecated': False}, + 'barr': {'id': 'Barr', 'deprecated': False}, + 'bcrypt-solar-designer': {'id': 'bcrypt-Solar-Designer', 'deprecated': False}, + 'beerware': {'id': 'Beerware', 'deprecated': False}, + 'bitstream-charter': {'id': 'Bitstream-Charter', 'deprecated': False}, + 'bitstream-vera': {'id': 'Bitstream-Vera', 'deprecated': False}, + 'bittorrent-1.0': {'id': 'BitTorrent-1.0', 'deprecated': False}, + 'bittorrent-1.1': {'id': 'BitTorrent-1.1', 'deprecated': False}, + 'blessing': {'id': 'blessing', 'deprecated': False}, + 'blueoak-1.0.0': {'id': 'BlueOak-1.0.0', 'deprecated': False}, + 'boehm-gc': {'id': 'Boehm-GC', 'deprecated': False}, + 'borceux': {'id': 'Borceux', 'deprecated': False}, + 'brian-gladman-2-clause': {'id': 'Brian-Gladman-2-Clause', 'deprecated': False}, + 'brian-gladman-3-clause': {'id': 'Brian-Gladman-3-Clause', 'deprecated': False}, + 'bsd-1-clause': {'id': 'BSD-1-Clause', 'deprecated': False}, + 'bsd-2-clause': {'id': 'BSD-2-Clause', 'deprecated': False}, + 'bsd-2-clause-darwin': {'id': 'BSD-2-Clause-Darwin', 'deprecated': False}, + 'bsd-2-clause-first-lines': {'id': 'BSD-2-Clause-first-lines', 'deprecated': False}, + 'bsd-2-clause-freebsd': {'id': 'BSD-2-Clause-FreeBSD', 'deprecated': True}, + 'bsd-2-clause-netbsd': {'id': 'BSD-2-Clause-NetBSD', 'deprecated': True}, + 'bsd-2-clause-patent': {'id': 'BSD-2-Clause-Patent', 'deprecated': False}, + 'bsd-2-clause-views': {'id': 'BSD-2-Clause-Views', 'deprecated': False}, + 'bsd-3-clause': {'id': 'BSD-3-Clause', 'deprecated': False}, + 'bsd-3-clause-acpica': {'id': 'BSD-3-Clause-acpica', 'deprecated': False}, + 'bsd-3-clause-attribution': {'id': 'BSD-3-Clause-Attribution', 'deprecated': False}, + 'bsd-3-clause-clear': {'id': 'BSD-3-Clause-Clear', 'deprecated': False}, + 'bsd-3-clause-flex': {'id': 'BSD-3-Clause-flex', 'deprecated': False}, + 'bsd-3-clause-hp': {'id': 'BSD-3-Clause-HP', 'deprecated': False}, + 'bsd-3-clause-lbnl': {'id': 'BSD-3-Clause-LBNL', 'deprecated': False}, + 'bsd-3-clause-modification': {'id': 'BSD-3-Clause-Modification', 'deprecated': False}, + 'bsd-3-clause-no-military-license': {'id': 'BSD-3-Clause-No-Military-License', 'deprecated': False}, + 'bsd-3-clause-no-nuclear-license': {'id': 'BSD-3-Clause-No-Nuclear-License', 'deprecated': False}, + 'bsd-3-clause-no-nuclear-license-2014': {'id': 'BSD-3-Clause-No-Nuclear-License-2014', 'deprecated': False}, + 'bsd-3-clause-no-nuclear-warranty': {'id': 'BSD-3-Clause-No-Nuclear-Warranty', 'deprecated': False}, + 'bsd-3-clause-open-mpi': {'id': 'BSD-3-Clause-Open-MPI', 'deprecated': False}, + 'bsd-3-clause-sun': {'id': 'BSD-3-Clause-Sun', 'deprecated': False}, + 'bsd-4-clause': {'id': 'BSD-4-Clause', 'deprecated': False}, + 'bsd-4-clause-shortened': {'id': 'BSD-4-Clause-Shortened', 'deprecated': False}, + 'bsd-4-clause-uc': {'id': 'BSD-4-Clause-UC', 'deprecated': False}, + 'bsd-4.3reno': {'id': 'BSD-4.3RENO', 'deprecated': False}, + 'bsd-4.3tahoe': {'id': 'BSD-4.3TAHOE', 'deprecated': False}, + 'bsd-advertising-acknowledgement': {'id': 'BSD-Advertising-Acknowledgement', 'deprecated': False}, + 'bsd-attribution-hpnd-disclaimer': {'id': 'BSD-Attribution-HPND-disclaimer', 'deprecated': False}, + 'bsd-inferno-nettverk': {'id': 'BSD-Inferno-Nettverk', 'deprecated': False}, + 'bsd-protection': {'id': 'BSD-Protection', 'deprecated': False}, + 'bsd-source-beginning-file': {'id': 'BSD-Source-beginning-file', 'deprecated': False}, + 'bsd-source-code': {'id': 'BSD-Source-Code', 'deprecated': False}, + 'bsd-systemics': {'id': 'BSD-Systemics', 'deprecated': False}, + 'bsd-systemics-w3works': {'id': 'BSD-Systemics-W3Works', 'deprecated': False}, + 'bsl-1.0': {'id': 'BSL-1.0', 'deprecated': False}, + 'busl-1.1': {'id': 'BUSL-1.1', 'deprecated': False}, + 'bzip2-1.0.5': {'id': 'bzip2-1.0.5', 'deprecated': True}, + 'bzip2-1.0.6': {'id': 'bzip2-1.0.6', 'deprecated': False}, + 'c-uda-1.0': {'id': 'C-UDA-1.0', 'deprecated': False}, + 'cal-1.0': {'id': 'CAL-1.0', 'deprecated': False}, + 'cal-1.0-combined-work-exception': {'id': 'CAL-1.0-Combined-Work-Exception', 'deprecated': False}, + 'caldera': {'id': 'Caldera', 'deprecated': False}, + 'caldera-no-preamble': {'id': 'Caldera-no-preamble', 'deprecated': False}, + 'catharon': {'id': 'Catharon', 'deprecated': False}, + 'catosl-1.1': {'id': 'CATOSL-1.1', 'deprecated': False}, + 'cc-by-1.0': {'id': 'CC-BY-1.0', 'deprecated': False}, + 'cc-by-2.0': {'id': 'CC-BY-2.0', 'deprecated': False}, + 'cc-by-2.5': {'id': 'CC-BY-2.5', 'deprecated': False}, + 'cc-by-2.5-au': {'id': 'CC-BY-2.5-AU', 'deprecated': False}, + 'cc-by-3.0': {'id': 'CC-BY-3.0', 'deprecated': False}, + 'cc-by-3.0-at': {'id': 'CC-BY-3.0-AT', 'deprecated': False}, + 'cc-by-3.0-au': {'id': 'CC-BY-3.0-AU', 'deprecated': False}, + 'cc-by-3.0-de': {'id': 'CC-BY-3.0-DE', 'deprecated': False}, + 'cc-by-3.0-igo': {'id': 'CC-BY-3.0-IGO', 'deprecated': False}, + 'cc-by-3.0-nl': {'id': 'CC-BY-3.0-NL', 'deprecated': False}, + 'cc-by-3.0-us': {'id': 'CC-BY-3.0-US', 'deprecated': False}, + 'cc-by-4.0': {'id': 'CC-BY-4.0', 'deprecated': False}, + 'cc-by-nc-1.0': {'id': 'CC-BY-NC-1.0', 'deprecated': False}, + 'cc-by-nc-2.0': {'id': 'CC-BY-NC-2.0', 'deprecated': False}, + 'cc-by-nc-2.5': {'id': 'CC-BY-NC-2.5', 'deprecated': False}, + 'cc-by-nc-3.0': {'id': 'CC-BY-NC-3.0', 'deprecated': False}, + 'cc-by-nc-3.0-de': {'id': 'CC-BY-NC-3.0-DE', 'deprecated': False}, + 'cc-by-nc-4.0': {'id': 'CC-BY-NC-4.0', 'deprecated': False}, + 'cc-by-nc-nd-1.0': {'id': 'CC-BY-NC-ND-1.0', 'deprecated': False}, + 'cc-by-nc-nd-2.0': {'id': 'CC-BY-NC-ND-2.0', 'deprecated': False}, + 'cc-by-nc-nd-2.5': {'id': 'CC-BY-NC-ND-2.5', 'deprecated': False}, + 'cc-by-nc-nd-3.0': {'id': 'CC-BY-NC-ND-3.0', 'deprecated': False}, + 'cc-by-nc-nd-3.0-de': {'id': 'CC-BY-NC-ND-3.0-DE', 'deprecated': False}, + 'cc-by-nc-nd-3.0-igo': {'id': 'CC-BY-NC-ND-3.0-IGO', 'deprecated': False}, + 'cc-by-nc-nd-4.0': {'id': 'CC-BY-NC-ND-4.0', 'deprecated': False}, + 'cc-by-nc-sa-1.0': {'id': 'CC-BY-NC-SA-1.0', 'deprecated': False}, + 'cc-by-nc-sa-2.0': {'id': 'CC-BY-NC-SA-2.0', 'deprecated': False}, + 'cc-by-nc-sa-2.0-de': {'id': 'CC-BY-NC-SA-2.0-DE', 'deprecated': False}, + 'cc-by-nc-sa-2.0-fr': {'id': 'CC-BY-NC-SA-2.0-FR', 'deprecated': False}, + 'cc-by-nc-sa-2.0-uk': {'id': 'CC-BY-NC-SA-2.0-UK', 'deprecated': False}, + 'cc-by-nc-sa-2.5': {'id': 'CC-BY-NC-SA-2.5', 'deprecated': False}, + 'cc-by-nc-sa-3.0': {'id': 'CC-BY-NC-SA-3.0', 'deprecated': False}, + 'cc-by-nc-sa-3.0-de': {'id': 'CC-BY-NC-SA-3.0-DE', 'deprecated': False}, + 'cc-by-nc-sa-3.0-igo': {'id': 'CC-BY-NC-SA-3.0-IGO', 'deprecated': False}, + 'cc-by-nc-sa-4.0': {'id': 'CC-BY-NC-SA-4.0', 'deprecated': False}, + 'cc-by-nd-1.0': {'id': 'CC-BY-ND-1.0', 'deprecated': False}, + 'cc-by-nd-2.0': {'id': 'CC-BY-ND-2.0', 'deprecated': False}, + 'cc-by-nd-2.5': {'id': 'CC-BY-ND-2.5', 'deprecated': False}, + 'cc-by-nd-3.0': {'id': 'CC-BY-ND-3.0', 'deprecated': False}, + 'cc-by-nd-3.0-de': {'id': 'CC-BY-ND-3.0-DE', 'deprecated': False}, + 'cc-by-nd-4.0': {'id': 'CC-BY-ND-4.0', 'deprecated': False}, + 'cc-by-sa-1.0': {'id': 'CC-BY-SA-1.0', 'deprecated': False}, + 'cc-by-sa-2.0': {'id': 'CC-BY-SA-2.0', 'deprecated': False}, + 'cc-by-sa-2.0-uk': {'id': 'CC-BY-SA-2.0-UK', 'deprecated': False}, + 'cc-by-sa-2.1-jp': {'id': 'CC-BY-SA-2.1-JP', 'deprecated': False}, + 'cc-by-sa-2.5': {'id': 'CC-BY-SA-2.5', 'deprecated': False}, + 'cc-by-sa-3.0': {'id': 'CC-BY-SA-3.0', 'deprecated': False}, + 'cc-by-sa-3.0-at': {'id': 'CC-BY-SA-3.0-AT', 'deprecated': False}, + 'cc-by-sa-3.0-de': {'id': 'CC-BY-SA-3.0-DE', 'deprecated': False}, + 'cc-by-sa-3.0-igo': {'id': 'CC-BY-SA-3.0-IGO', 'deprecated': False}, + 'cc-by-sa-4.0': {'id': 'CC-BY-SA-4.0', 'deprecated': False}, + 'cc-pddc': {'id': 'CC-PDDC', 'deprecated': False}, + 'cc0-1.0': {'id': 'CC0-1.0', 'deprecated': False}, + 'cddl-1.0': {'id': 'CDDL-1.0', 'deprecated': False}, + 'cddl-1.1': {'id': 'CDDL-1.1', 'deprecated': False}, + 'cdl-1.0': {'id': 'CDL-1.0', 'deprecated': False}, + 'cdla-permissive-1.0': {'id': 'CDLA-Permissive-1.0', 'deprecated': False}, + 'cdla-permissive-2.0': {'id': 'CDLA-Permissive-2.0', 'deprecated': False}, + 'cdla-sharing-1.0': {'id': 'CDLA-Sharing-1.0', 'deprecated': False}, + 'cecill-1.0': {'id': 'CECILL-1.0', 'deprecated': False}, + 'cecill-1.1': {'id': 'CECILL-1.1', 'deprecated': False}, + 'cecill-2.0': {'id': 'CECILL-2.0', 'deprecated': False}, + 'cecill-2.1': {'id': 'CECILL-2.1', 'deprecated': False}, + 'cecill-b': {'id': 'CECILL-B', 'deprecated': False}, + 'cecill-c': {'id': 'CECILL-C', 'deprecated': False}, + 'cern-ohl-1.1': {'id': 'CERN-OHL-1.1', 'deprecated': False}, + 'cern-ohl-1.2': {'id': 'CERN-OHL-1.2', 'deprecated': False}, + 'cern-ohl-p-2.0': {'id': 'CERN-OHL-P-2.0', 'deprecated': False}, + 'cern-ohl-s-2.0': {'id': 'CERN-OHL-S-2.0', 'deprecated': False}, + 'cern-ohl-w-2.0': {'id': 'CERN-OHL-W-2.0', 'deprecated': False}, + 'cfitsio': {'id': 'CFITSIO', 'deprecated': False}, + 'check-cvs': {'id': 'check-cvs', 'deprecated': False}, + 'checkmk': {'id': 'checkmk', 'deprecated': False}, + 'clartistic': {'id': 'ClArtistic', 'deprecated': False}, + 'clips': {'id': 'Clips', 'deprecated': False}, + 'cmu-mach': {'id': 'CMU-Mach', 'deprecated': False}, + 'cmu-mach-nodoc': {'id': 'CMU-Mach-nodoc', 'deprecated': False}, + 'cnri-jython': {'id': 'CNRI-Jython', 'deprecated': False}, + 'cnri-python': {'id': 'CNRI-Python', 'deprecated': False}, + 'cnri-python-gpl-compatible': {'id': 'CNRI-Python-GPL-Compatible', 'deprecated': False}, + 'coil-1.0': {'id': 'COIL-1.0', 'deprecated': False}, + 'community-spec-1.0': {'id': 'Community-Spec-1.0', 'deprecated': False}, + 'condor-1.1': {'id': 'Condor-1.1', 'deprecated': False}, + 'copyleft-next-0.3.0': {'id': 'copyleft-next-0.3.0', 'deprecated': False}, + 'copyleft-next-0.3.1': {'id': 'copyleft-next-0.3.1', 'deprecated': False}, + 'cornell-lossless-jpeg': {'id': 'Cornell-Lossless-JPEG', 'deprecated': False}, + 'cpal-1.0': {'id': 'CPAL-1.0', 'deprecated': False}, + 'cpl-1.0': {'id': 'CPL-1.0', 'deprecated': False}, + 'cpol-1.02': {'id': 'CPOL-1.02', 'deprecated': False}, + 'cronyx': {'id': 'Cronyx', 'deprecated': False}, + 'crossword': {'id': 'Crossword', 'deprecated': False}, + 'crystalstacker': {'id': 'CrystalStacker', 'deprecated': False}, + 'cua-opl-1.0': {'id': 'CUA-OPL-1.0', 'deprecated': False}, + 'cube': {'id': 'Cube', 'deprecated': False}, + 'curl': {'id': 'curl', 'deprecated': False}, + 'cve-tou': {'id': 'cve-tou', 'deprecated': False}, + 'd-fsl-1.0': {'id': 'D-FSL-1.0', 'deprecated': False}, + 'dec-3-clause': {'id': 'DEC-3-Clause', 'deprecated': False}, + 'diffmark': {'id': 'diffmark', 'deprecated': False}, + 'dl-de-by-2.0': {'id': 'DL-DE-BY-2.0', 'deprecated': False}, + 'dl-de-zero-2.0': {'id': 'DL-DE-ZERO-2.0', 'deprecated': False}, + 'doc': {'id': 'DOC', 'deprecated': False}, + 'docbook-schema': {'id': 'DocBook-Schema', 'deprecated': False}, + 'docbook-xml': {'id': 'DocBook-XML', 'deprecated': False}, + 'dotseqn': {'id': 'Dotseqn', 'deprecated': False}, + 'drl-1.0': {'id': 'DRL-1.0', 'deprecated': False}, + 'drl-1.1': {'id': 'DRL-1.1', 'deprecated': False}, + 'dsdp': {'id': 'DSDP', 'deprecated': False}, + 'dtoa': {'id': 'dtoa', 'deprecated': False}, + 'dvipdfm': {'id': 'dvipdfm', 'deprecated': False}, + 'ecl-1.0': {'id': 'ECL-1.0', 'deprecated': False}, + 'ecl-2.0': {'id': 'ECL-2.0', 'deprecated': False}, + 'ecos-2.0': {'id': 'eCos-2.0', 'deprecated': True}, + 'efl-1.0': {'id': 'EFL-1.0', 'deprecated': False}, + 'efl-2.0': {'id': 'EFL-2.0', 'deprecated': False}, + 'egenix': {'id': 'eGenix', 'deprecated': False}, + 'elastic-2.0': {'id': 'Elastic-2.0', 'deprecated': False}, + 'entessa': {'id': 'Entessa', 'deprecated': False}, + 'epics': {'id': 'EPICS', 'deprecated': False}, + 'epl-1.0': {'id': 'EPL-1.0', 'deprecated': False}, + 'epl-2.0': {'id': 'EPL-2.0', 'deprecated': False}, + 'erlpl-1.1': {'id': 'ErlPL-1.1', 'deprecated': False}, + 'etalab-2.0': {'id': 'etalab-2.0', 'deprecated': False}, + 'eudatagrid': {'id': 'EUDatagrid', 'deprecated': False}, + 'eupl-1.0': {'id': 'EUPL-1.0', 'deprecated': False}, + 'eupl-1.1': {'id': 'EUPL-1.1', 'deprecated': False}, + 'eupl-1.2': {'id': 'EUPL-1.2', 'deprecated': False}, + 'eurosym': {'id': 'Eurosym', 'deprecated': False}, + 'fair': {'id': 'Fair', 'deprecated': False}, + 'fbm': {'id': 'FBM', 'deprecated': False}, + 'fdk-aac': {'id': 'FDK-AAC', 'deprecated': False}, + 'ferguson-twofish': {'id': 'Ferguson-Twofish', 'deprecated': False}, + 'frameworx-1.0': {'id': 'Frameworx-1.0', 'deprecated': False}, + 'freebsd-doc': {'id': 'FreeBSD-DOC', 'deprecated': False}, + 'freeimage': {'id': 'FreeImage', 'deprecated': False}, + 'fsfap': {'id': 'FSFAP', 'deprecated': False}, + 'fsfap-no-warranty-disclaimer': {'id': 'FSFAP-no-warranty-disclaimer', 'deprecated': False}, + 'fsful': {'id': 'FSFUL', 'deprecated': False}, + 'fsfullr': {'id': 'FSFULLR', 'deprecated': False}, + 'fsfullrwd': {'id': 'FSFULLRWD', 'deprecated': False}, + 'ftl': {'id': 'FTL', 'deprecated': False}, + 'furuseth': {'id': 'Furuseth', 'deprecated': False}, + 'fwlw': {'id': 'fwlw', 'deprecated': False}, + 'gcr-docs': {'id': 'GCR-docs', 'deprecated': False}, + 'gd': {'id': 'GD', 'deprecated': False}, + 'gfdl-1.1': {'id': 'GFDL-1.1', 'deprecated': True}, + 'gfdl-1.1-invariants-only': {'id': 'GFDL-1.1-invariants-only', 'deprecated': False}, + 'gfdl-1.1-invariants-or-later': {'id': 'GFDL-1.1-invariants-or-later', 'deprecated': False}, + 'gfdl-1.1-no-invariants-only': {'id': 'GFDL-1.1-no-invariants-only', 'deprecated': False}, + 'gfdl-1.1-no-invariants-or-later': {'id': 'GFDL-1.1-no-invariants-or-later', 'deprecated': False}, + 'gfdl-1.1-only': {'id': 'GFDL-1.1-only', 'deprecated': False}, + 'gfdl-1.1-or-later': {'id': 'GFDL-1.1-or-later', 'deprecated': False}, + 'gfdl-1.2': {'id': 'GFDL-1.2', 'deprecated': True}, + 'gfdl-1.2-invariants-only': {'id': 'GFDL-1.2-invariants-only', 'deprecated': False}, + 'gfdl-1.2-invariants-or-later': {'id': 'GFDL-1.2-invariants-or-later', 'deprecated': False}, + 'gfdl-1.2-no-invariants-only': {'id': 'GFDL-1.2-no-invariants-only', 'deprecated': False}, + 'gfdl-1.2-no-invariants-or-later': {'id': 'GFDL-1.2-no-invariants-or-later', 'deprecated': False}, + 'gfdl-1.2-only': {'id': 'GFDL-1.2-only', 'deprecated': False}, + 'gfdl-1.2-or-later': {'id': 'GFDL-1.2-or-later', 'deprecated': False}, + 'gfdl-1.3': {'id': 'GFDL-1.3', 'deprecated': True}, + 'gfdl-1.3-invariants-only': {'id': 'GFDL-1.3-invariants-only', 'deprecated': False}, + 'gfdl-1.3-invariants-or-later': {'id': 'GFDL-1.3-invariants-or-later', 'deprecated': False}, + 'gfdl-1.3-no-invariants-only': {'id': 'GFDL-1.3-no-invariants-only', 'deprecated': False}, + 'gfdl-1.3-no-invariants-or-later': {'id': 'GFDL-1.3-no-invariants-or-later', 'deprecated': False}, + 'gfdl-1.3-only': {'id': 'GFDL-1.3-only', 'deprecated': False}, + 'gfdl-1.3-or-later': {'id': 'GFDL-1.3-or-later', 'deprecated': False}, + 'giftware': {'id': 'Giftware', 'deprecated': False}, + 'gl2ps': {'id': 'GL2PS', 'deprecated': False}, + 'glide': {'id': 'Glide', 'deprecated': False}, + 'glulxe': {'id': 'Glulxe', 'deprecated': False}, + 'glwtpl': {'id': 'GLWTPL', 'deprecated': False}, + 'gnuplot': {'id': 'gnuplot', 'deprecated': False}, + 'gpl-1.0': {'id': 'GPL-1.0', 'deprecated': True}, + 'gpl-1.0+': {'id': 'GPL-1.0+', 'deprecated': True}, + 'gpl-1.0-only': {'id': 'GPL-1.0-only', 'deprecated': False}, + 'gpl-1.0-or-later': {'id': 'GPL-1.0-or-later', 'deprecated': False}, + 'gpl-2.0': {'id': 'GPL-2.0', 'deprecated': True}, + 'gpl-2.0+': {'id': 'GPL-2.0+', 'deprecated': True}, + 'gpl-2.0-only': {'id': 'GPL-2.0-only', 'deprecated': False}, + 'gpl-2.0-or-later': {'id': 'GPL-2.0-or-later', 'deprecated': False}, + 'gpl-2.0-with-autoconf-exception': {'id': 'GPL-2.0-with-autoconf-exception', 'deprecated': True}, + 'gpl-2.0-with-bison-exception': {'id': 'GPL-2.0-with-bison-exception', 'deprecated': True}, + 'gpl-2.0-with-classpath-exception': {'id': 'GPL-2.0-with-classpath-exception', 'deprecated': True}, + 'gpl-2.0-with-font-exception': {'id': 'GPL-2.0-with-font-exception', 'deprecated': True}, + 'gpl-2.0-with-gcc-exception': {'id': 'GPL-2.0-with-GCC-exception', 'deprecated': True}, + 'gpl-3.0': {'id': 'GPL-3.0', 'deprecated': True}, + 'gpl-3.0+': {'id': 'GPL-3.0+', 'deprecated': True}, + 'gpl-3.0-only': {'id': 'GPL-3.0-only', 'deprecated': False}, + 'gpl-3.0-or-later': {'id': 'GPL-3.0-or-later', 'deprecated': False}, + 'gpl-3.0-with-autoconf-exception': {'id': 'GPL-3.0-with-autoconf-exception', 'deprecated': True}, + 'gpl-3.0-with-gcc-exception': {'id': 'GPL-3.0-with-GCC-exception', 'deprecated': True}, + 'graphics-gems': {'id': 'Graphics-Gems', 'deprecated': False}, + 'gsoap-1.3b': {'id': 'gSOAP-1.3b', 'deprecated': False}, + 'gtkbook': {'id': 'gtkbook', 'deprecated': False}, + 'gutmann': {'id': 'Gutmann', 'deprecated': False}, + 'haskellreport': {'id': 'HaskellReport', 'deprecated': False}, + 'hdparm': {'id': 'hdparm', 'deprecated': False}, + 'hidapi': {'id': 'HIDAPI', 'deprecated': False}, + 'hippocratic-2.1': {'id': 'Hippocratic-2.1', 'deprecated': False}, + 'hp-1986': {'id': 'HP-1986', 'deprecated': False}, + 'hp-1989': {'id': 'HP-1989', 'deprecated': False}, + 'hpnd': {'id': 'HPND', 'deprecated': False}, + 'hpnd-dec': {'id': 'HPND-DEC', 'deprecated': False}, + 'hpnd-doc': {'id': 'HPND-doc', 'deprecated': False}, + 'hpnd-doc-sell': {'id': 'HPND-doc-sell', 'deprecated': False}, + 'hpnd-export-us': {'id': 'HPND-export-US', 'deprecated': False}, + 'hpnd-export-us-acknowledgement': {'id': 'HPND-export-US-acknowledgement', 'deprecated': False}, + 'hpnd-export-us-modify': {'id': 'HPND-export-US-modify', 'deprecated': False}, + 'hpnd-export2-us': {'id': 'HPND-export2-US', 'deprecated': False}, + 'hpnd-fenneberg-livingston': {'id': 'HPND-Fenneberg-Livingston', 'deprecated': False}, + 'hpnd-inria-imag': {'id': 'HPND-INRIA-IMAG', 'deprecated': False}, + 'hpnd-intel': {'id': 'HPND-Intel', 'deprecated': False}, + 'hpnd-kevlin-henney': {'id': 'HPND-Kevlin-Henney', 'deprecated': False}, + 'hpnd-markus-kuhn': {'id': 'HPND-Markus-Kuhn', 'deprecated': False}, + 'hpnd-merchantability-variant': {'id': 'HPND-merchantability-variant', 'deprecated': False}, + 'hpnd-mit-disclaimer': {'id': 'HPND-MIT-disclaimer', 'deprecated': False}, + 'hpnd-netrek': {'id': 'HPND-Netrek', 'deprecated': False}, + 'hpnd-pbmplus': {'id': 'HPND-Pbmplus', 'deprecated': False}, + 'hpnd-sell-mit-disclaimer-xserver': {'id': 'HPND-sell-MIT-disclaimer-xserver', 'deprecated': False}, + 'hpnd-sell-regexpr': {'id': 'HPND-sell-regexpr', 'deprecated': False}, + 'hpnd-sell-variant': {'id': 'HPND-sell-variant', 'deprecated': False}, + 'hpnd-sell-variant-mit-disclaimer': {'id': 'HPND-sell-variant-MIT-disclaimer', 'deprecated': False}, + 'hpnd-sell-variant-mit-disclaimer-rev': {'id': 'HPND-sell-variant-MIT-disclaimer-rev', 'deprecated': False}, + 'hpnd-uc': {'id': 'HPND-UC', 'deprecated': False}, + 'hpnd-uc-export-us': {'id': 'HPND-UC-export-US', 'deprecated': False}, + 'htmltidy': {'id': 'HTMLTIDY', 'deprecated': False}, + 'ibm-pibs': {'id': 'IBM-pibs', 'deprecated': False}, + 'icu': {'id': 'ICU', 'deprecated': False}, + 'iec-code-components-eula': {'id': 'IEC-Code-Components-EULA', 'deprecated': False}, + 'ijg': {'id': 'IJG', 'deprecated': False}, + 'ijg-short': {'id': 'IJG-short', 'deprecated': False}, + 'imagemagick': {'id': 'ImageMagick', 'deprecated': False}, + 'imatix': {'id': 'iMatix', 'deprecated': False}, + 'imlib2': {'id': 'Imlib2', 'deprecated': False}, + 'info-zip': {'id': 'Info-ZIP', 'deprecated': False}, + 'inner-net-2.0': {'id': 'Inner-Net-2.0', 'deprecated': False}, + 'intel': {'id': 'Intel', 'deprecated': False}, + 'intel-acpi': {'id': 'Intel-ACPI', 'deprecated': False}, + 'interbase-1.0': {'id': 'Interbase-1.0', 'deprecated': False}, + 'ipa': {'id': 'IPA', 'deprecated': False}, + 'ipl-1.0': {'id': 'IPL-1.0', 'deprecated': False}, + 'isc': {'id': 'ISC', 'deprecated': False}, + 'isc-veillard': {'id': 'ISC-Veillard', 'deprecated': False}, + 'jam': {'id': 'Jam', 'deprecated': False}, + 'jasper-2.0': {'id': 'JasPer-2.0', 'deprecated': False}, + 'jpl-image': {'id': 'JPL-image', 'deprecated': False}, + 'jpnic': {'id': 'JPNIC', 'deprecated': False}, + 'json': {'id': 'JSON', 'deprecated': False}, + 'kastrup': {'id': 'Kastrup', 'deprecated': False}, + 'kazlib': {'id': 'Kazlib', 'deprecated': False}, + 'knuth-ctan': {'id': 'Knuth-CTAN', 'deprecated': False}, + 'lal-1.2': {'id': 'LAL-1.2', 'deprecated': False}, + 'lal-1.3': {'id': 'LAL-1.3', 'deprecated': False}, + 'latex2e': {'id': 'Latex2e', 'deprecated': False}, + 'latex2e-translated-notice': {'id': 'Latex2e-translated-notice', 'deprecated': False}, + 'leptonica': {'id': 'Leptonica', 'deprecated': False}, + 'lgpl-2.0': {'id': 'LGPL-2.0', 'deprecated': True}, + 'lgpl-2.0+': {'id': 'LGPL-2.0+', 'deprecated': True}, + 'lgpl-2.0-only': {'id': 'LGPL-2.0-only', 'deprecated': False}, + 'lgpl-2.0-or-later': {'id': 'LGPL-2.0-or-later', 'deprecated': False}, + 'lgpl-2.1': {'id': 'LGPL-2.1', 'deprecated': True}, + 'lgpl-2.1+': {'id': 'LGPL-2.1+', 'deprecated': True}, + 'lgpl-2.1-only': {'id': 'LGPL-2.1-only', 'deprecated': False}, + 'lgpl-2.1-or-later': {'id': 'LGPL-2.1-or-later', 'deprecated': False}, + 'lgpl-3.0': {'id': 'LGPL-3.0', 'deprecated': True}, + 'lgpl-3.0+': {'id': 'LGPL-3.0+', 'deprecated': True}, + 'lgpl-3.0-only': {'id': 'LGPL-3.0-only', 'deprecated': False}, + 'lgpl-3.0-or-later': {'id': 'LGPL-3.0-or-later', 'deprecated': False}, + 'lgpllr': {'id': 'LGPLLR', 'deprecated': False}, + 'libpng': {'id': 'Libpng', 'deprecated': False}, + 'libpng-2.0': {'id': 'libpng-2.0', 'deprecated': False}, + 'libselinux-1.0': {'id': 'libselinux-1.0', 'deprecated': False}, + 'libtiff': {'id': 'libtiff', 'deprecated': False}, + 'libutil-david-nugent': {'id': 'libutil-David-Nugent', 'deprecated': False}, + 'liliq-p-1.1': {'id': 'LiLiQ-P-1.1', 'deprecated': False}, + 'liliq-r-1.1': {'id': 'LiLiQ-R-1.1', 'deprecated': False}, + 'liliq-rplus-1.1': {'id': 'LiLiQ-Rplus-1.1', 'deprecated': False}, + 'linux-man-pages-1-para': {'id': 'Linux-man-pages-1-para', 'deprecated': False}, + 'linux-man-pages-copyleft': {'id': 'Linux-man-pages-copyleft', 'deprecated': False}, + 'linux-man-pages-copyleft-2-para': {'id': 'Linux-man-pages-copyleft-2-para', 'deprecated': False}, + 'linux-man-pages-copyleft-var': {'id': 'Linux-man-pages-copyleft-var', 'deprecated': False}, + 'linux-openib': {'id': 'Linux-OpenIB', 'deprecated': False}, + 'loop': {'id': 'LOOP', 'deprecated': False}, + 'lpd-document': {'id': 'LPD-document', 'deprecated': False}, + 'lpl-1.0': {'id': 'LPL-1.0', 'deprecated': False}, + 'lpl-1.02': {'id': 'LPL-1.02', 'deprecated': False}, + 'lppl-1.0': {'id': 'LPPL-1.0', 'deprecated': False}, + 'lppl-1.1': {'id': 'LPPL-1.1', 'deprecated': False}, + 'lppl-1.2': {'id': 'LPPL-1.2', 'deprecated': False}, + 'lppl-1.3a': {'id': 'LPPL-1.3a', 'deprecated': False}, + 'lppl-1.3c': {'id': 'LPPL-1.3c', 'deprecated': False}, + 'lsof': {'id': 'lsof', 'deprecated': False}, + 'lucida-bitmap-fonts': {'id': 'Lucida-Bitmap-Fonts', 'deprecated': False}, + 'lzma-sdk-9.11-to-9.20': {'id': 'LZMA-SDK-9.11-to-9.20', 'deprecated': False}, + 'lzma-sdk-9.22': {'id': 'LZMA-SDK-9.22', 'deprecated': False}, + 'mackerras-3-clause': {'id': 'Mackerras-3-Clause', 'deprecated': False}, + 'mackerras-3-clause-acknowledgment': {'id': 'Mackerras-3-Clause-acknowledgment', 'deprecated': False}, + 'magaz': {'id': 'magaz', 'deprecated': False}, + 'mailprio': {'id': 'mailprio', 'deprecated': False}, + 'makeindex': {'id': 'MakeIndex', 'deprecated': False}, + 'martin-birgmeier': {'id': 'Martin-Birgmeier', 'deprecated': False}, + 'mcphee-slideshow': {'id': 'McPhee-slideshow', 'deprecated': False}, + 'metamail': {'id': 'metamail', 'deprecated': False}, + 'minpack': {'id': 'Minpack', 'deprecated': False}, + 'miros': {'id': 'MirOS', 'deprecated': False}, + 'mit': {'id': 'MIT', 'deprecated': False}, + 'mit-0': {'id': 'MIT-0', 'deprecated': False}, + 'mit-advertising': {'id': 'MIT-advertising', 'deprecated': False}, + 'mit-cmu': {'id': 'MIT-CMU', 'deprecated': False}, + 'mit-enna': {'id': 'MIT-enna', 'deprecated': False}, + 'mit-feh': {'id': 'MIT-feh', 'deprecated': False}, + 'mit-festival': {'id': 'MIT-Festival', 'deprecated': False}, + 'mit-khronos-old': {'id': 'MIT-Khronos-old', 'deprecated': False}, + 'mit-modern-variant': {'id': 'MIT-Modern-Variant', 'deprecated': False}, + 'mit-open-group': {'id': 'MIT-open-group', 'deprecated': False}, + 'mit-testregex': {'id': 'MIT-testregex', 'deprecated': False}, + 'mit-wu': {'id': 'MIT-Wu', 'deprecated': False}, + 'mitnfa': {'id': 'MITNFA', 'deprecated': False}, + 'mmixware': {'id': 'MMIXware', 'deprecated': False}, + 'motosoto': {'id': 'Motosoto', 'deprecated': False}, + 'mpeg-ssg': {'id': 'MPEG-SSG', 'deprecated': False}, + 'mpi-permissive': {'id': 'mpi-permissive', 'deprecated': False}, + 'mpich2': {'id': 'mpich2', 'deprecated': False}, + 'mpl-1.0': {'id': 'MPL-1.0', 'deprecated': False}, + 'mpl-1.1': {'id': 'MPL-1.1', 'deprecated': False}, + 'mpl-2.0': {'id': 'MPL-2.0', 'deprecated': False}, + 'mpl-2.0-no-copyleft-exception': {'id': 'MPL-2.0-no-copyleft-exception', 'deprecated': False}, + 'mplus': {'id': 'mplus', 'deprecated': False}, + 'ms-lpl': {'id': 'MS-LPL', 'deprecated': False}, + 'ms-pl': {'id': 'MS-PL', 'deprecated': False}, + 'ms-rl': {'id': 'MS-RL', 'deprecated': False}, + 'mtll': {'id': 'MTLL', 'deprecated': False}, + 'mulanpsl-1.0': {'id': 'MulanPSL-1.0', 'deprecated': False}, + 'mulanpsl-2.0': {'id': 'MulanPSL-2.0', 'deprecated': False}, + 'multics': {'id': 'Multics', 'deprecated': False}, + 'mup': {'id': 'Mup', 'deprecated': False}, + 'naist-2003': {'id': 'NAIST-2003', 'deprecated': False}, + 'nasa-1.3': {'id': 'NASA-1.3', 'deprecated': False}, + 'naumen': {'id': 'Naumen', 'deprecated': False}, + 'nbpl-1.0': {'id': 'NBPL-1.0', 'deprecated': False}, + 'ncbi-pd': {'id': 'NCBI-PD', 'deprecated': False}, + 'ncgl-uk-2.0': {'id': 'NCGL-UK-2.0', 'deprecated': False}, + 'ncl': {'id': 'NCL', 'deprecated': False}, + 'ncsa': {'id': 'NCSA', 'deprecated': False}, + 'net-snmp': {'id': 'Net-SNMP', 'deprecated': True}, + 'netcdf': {'id': 'NetCDF', 'deprecated': False}, + 'newsletr': {'id': 'Newsletr', 'deprecated': False}, + 'ngpl': {'id': 'NGPL', 'deprecated': False}, + 'nicta-1.0': {'id': 'NICTA-1.0', 'deprecated': False}, + 'nist-pd': {'id': 'NIST-PD', 'deprecated': False}, + 'nist-pd-fallback': {'id': 'NIST-PD-fallback', 'deprecated': False}, + 'nist-software': {'id': 'NIST-Software', 'deprecated': False}, + 'nlod-1.0': {'id': 'NLOD-1.0', 'deprecated': False}, + 'nlod-2.0': {'id': 'NLOD-2.0', 'deprecated': False}, + 'nlpl': {'id': 'NLPL', 'deprecated': False}, + 'nokia': {'id': 'Nokia', 'deprecated': False}, + 'nosl': {'id': 'NOSL', 'deprecated': False}, + 'noweb': {'id': 'Noweb', 'deprecated': False}, + 'npl-1.0': {'id': 'NPL-1.0', 'deprecated': False}, + 'npl-1.1': {'id': 'NPL-1.1', 'deprecated': False}, + 'nposl-3.0': {'id': 'NPOSL-3.0', 'deprecated': False}, + 'nrl': {'id': 'NRL', 'deprecated': False}, + 'ntp': {'id': 'NTP', 'deprecated': False}, + 'ntp-0': {'id': 'NTP-0', 'deprecated': False}, + 'nunit': {'id': 'Nunit', 'deprecated': True}, + 'o-uda-1.0': {'id': 'O-UDA-1.0', 'deprecated': False}, + 'oar': {'id': 'OAR', 'deprecated': False}, + 'occt-pl': {'id': 'OCCT-PL', 'deprecated': False}, + 'oclc-2.0': {'id': 'OCLC-2.0', 'deprecated': False}, + 'odbl-1.0': {'id': 'ODbL-1.0', 'deprecated': False}, + 'odc-by-1.0': {'id': 'ODC-By-1.0', 'deprecated': False}, + 'offis': {'id': 'OFFIS', 'deprecated': False}, + 'ofl-1.0': {'id': 'OFL-1.0', 'deprecated': False}, + 'ofl-1.0-no-rfn': {'id': 'OFL-1.0-no-RFN', 'deprecated': False}, + 'ofl-1.0-rfn': {'id': 'OFL-1.0-RFN', 'deprecated': False}, + 'ofl-1.1': {'id': 'OFL-1.1', 'deprecated': False}, + 'ofl-1.1-no-rfn': {'id': 'OFL-1.1-no-RFN', 'deprecated': False}, + 'ofl-1.1-rfn': {'id': 'OFL-1.1-RFN', 'deprecated': False}, + 'ogc-1.0': {'id': 'OGC-1.0', 'deprecated': False}, + 'ogdl-taiwan-1.0': {'id': 'OGDL-Taiwan-1.0', 'deprecated': False}, + 'ogl-canada-2.0': {'id': 'OGL-Canada-2.0', 'deprecated': False}, + 'ogl-uk-1.0': {'id': 'OGL-UK-1.0', 'deprecated': False}, + 'ogl-uk-2.0': {'id': 'OGL-UK-2.0', 'deprecated': False}, + 'ogl-uk-3.0': {'id': 'OGL-UK-3.0', 'deprecated': False}, + 'ogtsl': {'id': 'OGTSL', 'deprecated': False}, + 'oldap-1.1': {'id': 'OLDAP-1.1', 'deprecated': False}, + 'oldap-1.2': {'id': 'OLDAP-1.2', 'deprecated': False}, + 'oldap-1.3': {'id': 'OLDAP-1.3', 'deprecated': False}, + 'oldap-1.4': {'id': 'OLDAP-1.4', 'deprecated': False}, + 'oldap-2.0': {'id': 'OLDAP-2.0', 'deprecated': False}, + 'oldap-2.0.1': {'id': 'OLDAP-2.0.1', 'deprecated': False}, + 'oldap-2.1': {'id': 'OLDAP-2.1', 'deprecated': False}, + 'oldap-2.2': {'id': 'OLDAP-2.2', 'deprecated': False}, + 'oldap-2.2.1': {'id': 'OLDAP-2.2.1', 'deprecated': False}, + 'oldap-2.2.2': {'id': 'OLDAP-2.2.2', 'deprecated': False}, + 'oldap-2.3': {'id': 'OLDAP-2.3', 'deprecated': False}, + 'oldap-2.4': {'id': 'OLDAP-2.4', 'deprecated': False}, + 'oldap-2.5': {'id': 'OLDAP-2.5', 'deprecated': False}, + 'oldap-2.6': {'id': 'OLDAP-2.6', 'deprecated': False}, + 'oldap-2.7': {'id': 'OLDAP-2.7', 'deprecated': False}, + 'oldap-2.8': {'id': 'OLDAP-2.8', 'deprecated': False}, + 'olfl-1.3': {'id': 'OLFL-1.3', 'deprecated': False}, + 'oml': {'id': 'OML', 'deprecated': False}, + 'openpbs-2.3': {'id': 'OpenPBS-2.3', 'deprecated': False}, + 'openssl': {'id': 'OpenSSL', 'deprecated': False}, + 'openssl-standalone': {'id': 'OpenSSL-standalone', 'deprecated': False}, + 'openvision': {'id': 'OpenVision', 'deprecated': False}, + 'opl-1.0': {'id': 'OPL-1.0', 'deprecated': False}, + 'opl-uk-3.0': {'id': 'OPL-UK-3.0', 'deprecated': False}, + 'opubl-1.0': {'id': 'OPUBL-1.0', 'deprecated': False}, + 'oset-pl-2.1': {'id': 'OSET-PL-2.1', 'deprecated': False}, + 'osl-1.0': {'id': 'OSL-1.0', 'deprecated': False}, + 'osl-1.1': {'id': 'OSL-1.1', 'deprecated': False}, + 'osl-2.0': {'id': 'OSL-2.0', 'deprecated': False}, + 'osl-2.1': {'id': 'OSL-2.1', 'deprecated': False}, + 'osl-3.0': {'id': 'OSL-3.0', 'deprecated': False}, + 'padl': {'id': 'PADL', 'deprecated': False}, + 'parity-6.0.0': {'id': 'Parity-6.0.0', 'deprecated': False}, + 'parity-7.0.0': {'id': 'Parity-7.0.0', 'deprecated': False}, + 'pddl-1.0': {'id': 'PDDL-1.0', 'deprecated': False}, + 'php-3.0': {'id': 'PHP-3.0', 'deprecated': False}, + 'php-3.01': {'id': 'PHP-3.01', 'deprecated': False}, + 'pixar': {'id': 'Pixar', 'deprecated': False}, + 'pkgconf': {'id': 'pkgconf', 'deprecated': False}, + 'plexus': {'id': 'Plexus', 'deprecated': False}, + 'pnmstitch': {'id': 'pnmstitch', 'deprecated': False}, + 'polyform-noncommercial-1.0.0': {'id': 'PolyForm-Noncommercial-1.0.0', 'deprecated': False}, + 'polyform-small-business-1.0.0': {'id': 'PolyForm-Small-Business-1.0.0', 'deprecated': False}, + 'postgresql': {'id': 'PostgreSQL', 'deprecated': False}, + 'ppl': {'id': 'PPL', 'deprecated': False}, + 'psf-2.0': {'id': 'PSF-2.0', 'deprecated': False}, + 'psfrag': {'id': 'psfrag', 'deprecated': False}, + 'psutils': {'id': 'psutils', 'deprecated': False}, + 'python-2.0': {'id': 'Python-2.0', 'deprecated': False}, + 'python-2.0.1': {'id': 'Python-2.0.1', 'deprecated': False}, + 'python-ldap': {'id': 'python-ldap', 'deprecated': False}, + 'qhull': {'id': 'Qhull', 'deprecated': False}, + 'qpl-1.0': {'id': 'QPL-1.0', 'deprecated': False}, + 'qpl-1.0-inria-2004': {'id': 'QPL-1.0-INRIA-2004', 'deprecated': False}, + 'radvd': {'id': 'radvd', 'deprecated': False}, + 'rdisc': {'id': 'Rdisc', 'deprecated': False}, + 'rhecos-1.1': {'id': 'RHeCos-1.1', 'deprecated': False}, + 'rpl-1.1': {'id': 'RPL-1.1', 'deprecated': False}, + 'rpl-1.5': {'id': 'RPL-1.5', 'deprecated': False}, + 'rpsl-1.0': {'id': 'RPSL-1.0', 'deprecated': False}, + 'rsa-md': {'id': 'RSA-MD', 'deprecated': False}, + 'rscpl': {'id': 'RSCPL', 'deprecated': False}, + 'ruby': {'id': 'Ruby', 'deprecated': False}, + 'ruby-pty': {'id': 'Ruby-pty', 'deprecated': False}, + 'sax-pd': {'id': 'SAX-PD', 'deprecated': False}, + 'sax-pd-2.0': {'id': 'SAX-PD-2.0', 'deprecated': False}, + 'saxpath': {'id': 'Saxpath', 'deprecated': False}, + 'scea': {'id': 'SCEA', 'deprecated': False}, + 'schemereport': {'id': 'SchemeReport', 'deprecated': False}, + 'sendmail': {'id': 'Sendmail', 'deprecated': False}, + 'sendmail-8.23': {'id': 'Sendmail-8.23', 'deprecated': False}, + 'sgi-b-1.0': {'id': 'SGI-B-1.0', 'deprecated': False}, + 'sgi-b-1.1': {'id': 'SGI-B-1.1', 'deprecated': False}, + 'sgi-b-2.0': {'id': 'SGI-B-2.0', 'deprecated': False}, + 'sgi-opengl': {'id': 'SGI-OpenGL', 'deprecated': False}, + 'sgp4': {'id': 'SGP4', 'deprecated': False}, + 'shl-0.5': {'id': 'SHL-0.5', 'deprecated': False}, + 'shl-0.51': {'id': 'SHL-0.51', 'deprecated': False}, + 'simpl-2.0': {'id': 'SimPL-2.0', 'deprecated': False}, + 'sissl': {'id': 'SISSL', 'deprecated': False}, + 'sissl-1.2': {'id': 'SISSL-1.2', 'deprecated': False}, + 'sl': {'id': 'SL', 'deprecated': False}, + 'sleepycat': {'id': 'Sleepycat', 'deprecated': False}, + 'smlnj': {'id': 'SMLNJ', 'deprecated': False}, + 'smppl': {'id': 'SMPPL', 'deprecated': False}, + 'snia': {'id': 'SNIA', 'deprecated': False}, + 'snprintf': {'id': 'snprintf', 'deprecated': False}, + 'softsurfer': {'id': 'softSurfer', 'deprecated': False}, + 'soundex': {'id': 'Soundex', 'deprecated': False}, + 'spencer-86': {'id': 'Spencer-86', 'deprecated': False}, + 'spencer-94': {'id': 'Spencer-94', 'deprecated': False}, + 'spencer-99': {'id': 'Spencer-99', 'deprecated': False}, + 'spl-1.0': {'id': 'SPL-1.0', 'deprecated': False}, + 'ssh-keyscan': {'id': 'ssh-keyscan', 'deprecated': False}, + 'ssh-openssh': {'id': 'SSH-OpenSSH', 'deprecated': False}, + 'ssh-short': {'id': 'SSH-short', 'deprecated': False}, + 'ssleay-standalone': {'id': 'SSLeay-standalone', 'deprecated': False}, + 'sspl-1.0': {'id': 'SSPL-1.0', 'deprecated': False}, + 'standardml-nj': {'id': 'StandardML-NJ', 'deprecated': True}, + 'sugarcrm-1.1.3': {'id': 'SugarCRM-1.1.3', 'deprecated': False}, + 'sun-ppp': {'id': 'Sun-PPP', 'deprecated': False}, + 'sun-ppp-2000': {'id': 'Sun-PPP-2000', 'deprecated': False}, + 'sunpro': {'id': 'SunPro', 'deprecated': False}, + 'swl': {'id': 'SWL', 'deprecated': False}, + 'swrule': {'id': 'swrule', 'deprecated': False}, + 'symlinks': {'id': 'Symlinks', 'deprecated': False}, + 'tapr-ohl-1.0': {'id': 'TAPR-OHL-1.0', 'deprecated': False}, + 'tcl': {'id': 'TCL', 'deprecated': False}, + 'tcp-wrappers': {'id': 'TCP-wrappers', 'deprecated': False}, + 'termreadkey': {'id': 'TermReadKey', 'deprecated': False}, + 'tgppl-1.0': {'id': 'TGPPL-1.0', 'deprecated': False}, + 'threeparttable': {'id': 'threeparttable', 'deprecated': False}, + 'tmate': {'id': 'TMate', 'deprecated': False}, + 'torque-1.1': {'id': 'TORQUE-1.1', 'deprecated': False}, + 'tosl': {'id': 'TOSL', 'deprecated': False}, + 'tpdl': {'id': 'TPDL', 'deprecated': False}, + 'tpl-1.0': {'id': 'TPL-1.0', 'deprecated': False}, + 'ttwl': {'id': 'TTWL', 'deprecated': False}, + 'ttyp0': {'id': 'TTYP0', 'deprecated': False}, + 'tu-berlin-1.0': {'id': 'TU-Berlin-1.0', 'deprecated': False}, + 'tu-berlin-2.0': {'id': 'TU-Berlin-2.0', 'deprecated': False}, + 'ubuntu-font-1.0': {'id': 'Ubuntu-font-1.0', 'deprecated': False}, + 'ucar': {'id': 'UCAR', 'deprecated': False}, + 'ucl-1.0': {'id': 'UCL-1.0', 'deprecated': False}, + 'ulem': {'id': 'ulem', 'deprecated': False}, + 'umich-merit': {'id': 'UMich-Merit', 'deprecated': False}, + 'unicode-3.0': {'id': 'Unicode-3.0', 'deprecated': False}, + 'unicode-dfs-2015': {'id': 'Unicode-DFS-2015', 'deprecated': False}, + 'unicode-dfs-2016': {'id': 'Unicode-DFS-2016', 'deprecated': False}, + 'unicode-tou': {'id': 'Unicode-TOU', 'deprecated': False}, + 'unixcrypt': {'id': 'UnixCrypt', 'deprecated': False}, + 'unlicense': {'id': 'Unlicense', 'deprecated': False}, + 'upl-1.0': {'id': 'UPL-1.0', 'deprecated': False}, + 'urt-rle': {'id': 'URT-RLE', 'deprecated': False}, + 'vim': {'id': 'Vim', 'deprecated': False}, + 'vostrom': {'id': 'VOSTROM', 'deprecated': False}, + 'vsl-1.0': {'id': 'VSL-1.0', 'deprecated': False}, + 'w3c': {'id': 'W3C', 'deprecated': False}, + 'w3c-19980720': {'id': 'W3C-19980720', 'deprecated': False}, + 'w3c-20150513': {'id': 'W3C-20150513', 'deprecated': False}, + 'w3m': {'id': 'w3m', 'deprecated': False}, + 'watcom-1.0': {'id': 'Watcom-1.0', 'deprecated': False}, + 'widget-workshop': {'id': 'Widget-Workshop', 'deprecated': False}, + 'wsuipa': {'id': 'Wsuipa', 'deprecated': False}, + 'wtfpl': {'id': 'WTFPL', 'deprecated': False}, + 'wxwindows': {'id': 'wxWindows', 'deprecated': True}, + 'x11': {'id': 'X11', 'deprecated': False}, + 'x11-distribute-modifications-variant': {'id': 'X11-distribute-modifications-variant', 'deprecated': False}, + 'x11-swapped': {'id': 'X11-swapped', 'deprecated': False}, + 'xdebug-1.03': {'id': 'Xdebug-1.03', 'deprecated': False}, + 'xerox': {'id': 'Xerox', 'deprecated': False}, + 'xfig': {'id': 'Xfig', 'deprecated': False}, + 'xfree86-1.1': {'id': 'XFree86-1.1', 'deprecated': False}, + 'xinetd': {'id': 'xinetd', 'deprecated': False}, + 'xkeyboard-config-zinoviev': {'id': 'xkeyboard-config-Zinoviev', 'deprecated': False}, + 'xlock': {'id': 'xlock', 'deprecated': False}, + 'xnet': {'id': 'Xnet', 'deprecated': False}, + 'xpp': {'id': 'xpp', 'deprecated': False}, + 'xskat': {'id': 'XSkat', 'deprecated': False}, + 'xzoom': {'id': 'xzoom', 'deprecated': False}, + 'ypl-1.0': {'id': 'YPL-1.0', 'deprecated': False}, + 'ypl-1.1': {'id': 'YPL-1.1', 'deprecated': False}, + 'zed': {'id': 'Zed', 'deprecated': False}, + 'zeeff': {'id': 'Zeeff', 'deprecated': False}, + 'zend-2.0': {'id': 'Zend-2.0', 'deprecated': False}, + 'zimbra-1.3': {'id': 'Zimbra-1.3', 'deprecated': False}, + 'zimbra-1.4': {'id': 'Zimbra-1.4', 'deprecated': False}, + 'zlib': {'id': 'Zlib', 'deprecated': False}, + 'zlib-acknowledgement': {'id': 'zlib-acknowledgement', 'deprecated': False}, + 'zpl-1.1': {'id': 'ZPL-1.1', 'deprecated': False}, + 'zpl-2.0': {'id': 'ZPL-2.0', 'deprecated': False}, + 'zpl-2.1': {'id': 'ZPL-2.1', 'deprecated': False}, +} + +EXCEPTIONS: dict[str, SPDXException] = { + '389-exception': {'id': '389-exception', 'deprecated': False}, + 'asterisk-exception': {'id': 'Asterisk-exception', 'deprecated': False}, + 'asterisk-linking-protocols-exception': {'id': 'Asterisk-linking-protocols-exception', 'deprecated': False}, + 'autoconf-exception-2.0': {'id': 'Autoconf-exception-2.0', 'deprecated': False}, + 'autoconf-exception-3.0': {'id': 'Autoconf-exception-3.0', 'deprecated': False}, + 'autoconf-exception-generic': {'id': 'Autoconf-exception-generic', 'deprecated': False}, + 'autoconf-exception-generic-3.0': {'id': 'Autoconf-exception-generic-3.0', 'deprecated': False}, + 'autoconf-exception-macro': {'id': 'Autoconf-exception-macro', 'deprecated': False}, + 'bison-exception-1.24': {'id': 'Bison-exception-1.24', 'deprecated': False}, + 'bison-exception-2.2': {'id': 'Bison-exception-2.2', 'deprecated': False}, + 'bootloader-exception': {'id': 'Bootloader-exception', 'deprecated': False}, + 'classpath-exception-2.0': {'id': 'Classpath-exception-2.0', 'deprecated': False}, + 'clisp-exception-2.0': {'id': 'CLISP-exception-2.0', 'deprecated': False}, + 'cryptsetup-openssl-exception': {'id': 'cryptsetup-OpenSSL-exception', 'deprecated': False}, + 'digirule-foss-exception': {'id': 'DigiRule-FOSS-exception', 'deprecated': False}, + 'ecos-exception-2.0': {'id': 'eCos-exception-2.0', 'deprecated': False}, + 'erlang-otp-linking-exception': {'id': 'erlang-otp-linking-exception', 'deprecated': False}, + 'fawkes-runtime-exception': {'id': 'Fawkes-Runtime-exception', 'deprecated': False}, + 'fltk-exception': {'id': 'FLTK-exception', 'deprecated': False}, + 'fmt-exception': {'id': 'fmt-exception', 'deprecated': False}, + 'font-exception-2.0': {'id': 'Font-exception-2.0', 'deprecated': False}, + 'freertos-exception-2.0': {'id': 'freertos-exception-2.0', 'deprecated': False}, + 'gcc-exception-2.0': {'id': 'GCC-exception-2.0', 'deprecated': False}, + 'gcc-exception-2.0-note': {'id': 'GCC-exception-2.0-note', 'deprecated': False}, + 'gcc-exception-3.1': {'id': 'GCC-exception-3.1', 'deprecated': False}, + 'gmsh-exception': {'id': 'Gmsh-exception', 'deprecated': False}, + 'gnat-exception': {'id': 'GNAT-exception', 'deprecated': False}, + 'gnome-examples-exception': {'id': 'GNOME-examples-exception', 'deprecated': False}, + 'gnu-compiler-exception': {'id': 'GNU-compiler-exception', 'deprecated': False}, + 'gnu-javamail-exception': {'id': 'gnu-javamail-exception', 'deprecated': False}, + 'gpl-3.0-interface-exception': {'id': 'GPL-3.0-interface-exception', 'deprecated': False}, + 'gpl-3.0-linking-exception': {'id': 'GPL-3.0-linking-exception', 'deprecated': False}, + 'gpl-3.0-linking-source-exception': {'id': 'GPL-3.0-linking-source-exception', 'deprecated': False}, + 'gpl-cc-1.0': {'id': 'GPL-CC-1.0', 'deprecated': False}, + 'gstreamer-exception-2005': {'id': 'GStreamer-exception-2005', 'deprecated': False}, + 'gstreamer-exception-2008': {'id': 'GStreamer-exception-2008', 'deprecated': False}, + 'i2p-gpl-java-exception': {'id': 'i2p-gpl-java-exception', 'deprecated': False}, + 'kicad-libraries-exception': {'id': 'KiCad-libraries-exception', 'deprecated': False}, + 'lgpl-3.0-linking-exception': {'id': 'LGPL-3.0-linking-exception', 'deprecated': False}, + 'libpri-openh323-exception': {'id': 'libpri-OpenH323-exception', 'deprecated': False}, + 'libtool-exception': {'id': 'Libtool-exception', 'deprecated': False}, + 'linux-syscall-note': {'id': 'Linux-syscall-note', 'deprecated': False}, + 'llgpl': {'id': 'LLGPL', 'deprecated': False}, + 'llvm-exception': {'id': 'LLVM-exception', 'deprecated': False}, + 'lzma-exception': {'id': 'LZMA-exception', 'deprecated': False}, + 'mif-exception': {'id': 'mif-exception', 'deprecated': False}, + 'nokia-qt-exception-1.1': {'id': 'Nokia-Qt-exception-1.1', 'deprecated': True}, + 'ocaml-lgpl-linking-exception': {'id': 'OCaml-LGPL-linking-exception', 'deprecated': False}, + 'occt-exception-1.0': {'id': 'OCCT-exception-1.0', 'deprecated': False}, + 'openjdk-assembly-exception-1.0': {'id': 'OpenJDK-assembly-exception-1.0', 'deprecated': False}, + 'openvpn-openssl-exception': {'id': 'openvpn-openssl-exception', 'deprecated': False}, + 'pcre2-exception': {'id': 'PCRE2-exception', 'deprecated': False}, + 'ps-or-pdf-font-exception-20170817': {'id': 'PS-or-PDF-font-exception-20170817', 'deprecated': False}, + 'qpl-1.0-inria-2004-exception': {'id': 'QPL-1.0-INRIA-2004-exception', 'deprecated': False}, + 'qt-gpl-exception-1.0': {'id': 'Qt-GPL-exception-1.0', 'deprecated': False}, + 'qt-lgpl-exception-1.1': {'id': 'Qt-LGPL-exception-1.1', 'deprecated': False}, + 'qwt-exception-1.0': {'id': 'Qwt-exception-1.0', 'deprecated': False}, + 'romic-exception': {'id': 'romic-exception', 'deprecated': False}, + 'rrdtool-floss-exception-2.0': {'id': 'RRDtool-FLOSS-exception-2.0', 'deprecated': False}, + 'sane-exception': {'id': 'SANE-exception', 'deprecated': False}, + 'shl-2.0': {'id': 'SHL-2.0', 'deprecated': False}, + 'shl-2.1': {'id': 'SHL-2.1', 'deprecated': False}, + 'stunnel-exception': {'id': 'stunnel-exception', 'deprecated': False}, + 'swi-exception': {'id': 'SWI-exception', 'deprecated': False}, + 'swift-exception': {'id': 'Swift-exception', 'deprecated': False}, + 'texinfo-exception': {'id': 'Texinfo-exception', 'deprecated': False}, + 'u-boot-exception-2.0': {'id': 'u-boot-exception-2.0', 'deprecated': False}, + 'ubdl-exception': {'id': 'UBDL-exception', 'deprecated': False}, + 'universal-foss-exception-1.0': {'id': 'Universal-FOSS-exception-1.0', 'deprecated': False}, + 'vsftpd-openssl-exception': {'id': 'vsftpd-openssl-exception', 'deprecated': False}, + 'wxwindows-exception-3.1': {'id': 'WxWindows-exception-3.1', 'deprecated': False}, + 'x11vnc-openssl-exception': {'id': 'x11vnc-openssl-exception', 'deprecated': False}, +} diff --git a/lib/python3.1/site-packages/packaging/markers.py b/lib/python3.1/site-packages/packaging/markers.py new file mode 100644 index 0000000000000000000000000000000000000000..e7cea57297a23809548fcd0d344740796bae945b --- /dev/null +++ b/lib/python3.1/site-packages/packaging/markers.py @@ -0,0 +1,362 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import operator +import os +import platform +import sys +from typing import AbstractSet, Any, Callable, Literal, TypedDict, Union, cast + +from ._parser import MarkerAtom, MarkerList, Op, Value, Variable +from ._parser import parse_marker as _parse_marker +from ._tokenizer import ParserSyntaxError +from .specifiers import InvalidSpecifier, Specifier +from .utils import canonicalize_name + +__all__ = [ + "EvaluateContext", + "InvalidMarker", + "Marker", + "UndefinedComparison", + "UndefinedEnvironmentName", + "default_environment", +] + +Operator = Callable[[str, Union[str, AbstractSet[str]]], bool] +EvaluateContext = Literal["metadata", "lock_file", "requirement"] +MARKERS_ALLOWING_SET = {"extras", "dependency_groups"} + + +class InvalidMarker(ValueError): + """ + An invalid marker was found, users should refer to PEP 508. + """ + + +class UndefinedComparison(ValueError): + """ + An invalid operation was attempted on a value that doesn't support it. + """ + + +class UndefinedEnvironmentName(ValueError): + """ + A name was attempted to be used that does not exist inside of the + environment. + """ + + +class Environment(TypedDict): + implementation_name: str + """The implementation's identifier, e.g. ``'cpython'``.""" + + implementation_version: str + """ + The implementation's version, e.g. ``'3.13.0a2'`` for CPython 3.13.0a2, or + ``'7.3.13'`` for PyPy3.10 v7.3.13. + """ + + os_name: str + """ + The value of :py:data:`os.name`. The name of the operating system dependent module + imported, e.g. ``'posix'``. + """ + + platform_machine: str + """ + Returns the machine type, e.g. ``'i386'``. + + An empty string if the value cannot be determined. + """ + + platform_release: str + """ + The system's release, e.g. ``'2.2.0'`` or ``'NT'``. + + An empty string if the value cannot be determined. + """ + + platform_system: str + """ + The system/OS name, e.g. ``'Linux'``, ``'Windows'`` or ``'Java'``. + + An empty string if the value cannot be determined. + """ + + platform_version: str + """ + The system's release version, e.g. ``'#3 on degas'``. + + An empty string if the value cannot be determined. + """ + + python_full_version: str + """ + The Python version as string ``'major.minor.patchlevel'``. + + Note that unlike the Python :py:data:`sys.version`, this value will always include + the patchlevel (it defaults to 0). + """ + + platform_python_implementation: str + """ + A string identifying the Python implementation, e.g. ``'CPython'``. + """ + + python_version: str + """The Python version as string ``'major.minor'``.""" + + sys_platform: str + """ + This string contains a platform identifier that can be used to append + platform-specific components to :py:data:`sys.path`, for instance. + + For Unix systems, except on Linux and AIX, this is the lowercased OS name as + returned by ``uname -s`` with the first part of the version as returned by + ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, at the time when Python + was built. + """ + + +def _normalize_extra_values(results: Any) -> Any: + """ + Normalize extra values. + """ + if isinstance(results[0], tuple): + lhs, op, rhs = results[0] + if isinstance(lhs, Variable) and lhs.value == "extra": + normalized_extra = canonicalize_name(rhs.value) + rhs = Value(normalized_extra) + elif isinstance(rhs, Variable) and rhs.value == "extra": + normalized_extra = canonicalize_name(lhs.value) + lhs = Value(normalized_extra) + results[0] = lhs, op, rhs + return results + + +def _format_marker( + marker: list[str] | MarkerAtom | str, first: bool | None = True +) -> str: + assert isinstance(marker, (list, tuple, str)) + + # Sometimes we have a structure like [[...]] which is a single item list + # where the single item is itself it's own list. In that case we want skip + # the rest of this function so that we don't get extraneous () on the + # outside. + if ( + isinstance(marker, list) + and len(marker) == 1 + and isinstance(marker[0], (list, tuple)) + ): + return _format_marker(marker[0]) + + if isinstance(marker, list): + inner = (_format_marker(m, first=False) for m in marker) + if first: + return " ".join(inner) + else: + return "(" + " ".join(inner) + ")" + elif isinstance(marker, tuple): + return " ".join([m.serialize() for m in marker]) + else: + return marker + + +_operators: dict[str, Operator] = { + "in": lambda lhs, rhs: lhs in rhs, + "not in": lambda lhs, rhs: lhs not in rhs, + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, +} + + +def _eval_op(lhs: str, op: Op, rhs: str | AbstractSet[str]) -> bool: + if isinstance(rhs, str): + try: + spec = Specifier("".join([op.serialize(), rhs])) + except InvalidSpecifier: + pass + else: + return spec.contains(lhs, prereleases=True) + + oper: Operator | None = _operators.get(op.serialize()) + if oper is None: + raise UndefinedComparison(f"Undefined {op!r} on {lhs!r} and {rhs!r}.") + + return oper(lhs, rhs) + + +def _normalize( + lhs: str, rhs: str | AbstractSet[str], key: str +) -> tuple[str, str | AbstractSet[str]]: + # PEP 685 – Comparison of extra names for optional distribution dependencies + # https://peps.python.org/pep-0685/ + # > When comparing extra names, tools MUST normalize the names being + # > compared using the semantics outlined in PEP 503 for names + if key == "extra": + assert isinstance(rhs, str), "extra value must be a string" + return (canonicalize_name(lhs), canonicalize_name(rhs)) + if key in MARKERS_ALLOWING_SET: + if isinstance(rhs, str): # pragma: no cover + return (canonicalize_name(lhs), canonicalize_name(rhs)) + else: + return (canonicalize_name(lhs), {canonicalize_name(v) for v in rhs}) + + # other environment markers don't have such standards + return lhs, rhs + + +def _evaluate_markers( + markers: MarkerList, environment: dict[str, str | AbstractSet[str]] +) -> bool: + groups: list[list[bool]] = [[]] + + for marker in markers: + assert isinstance(marker, (list, tuple, str)) + + if isinstance(marker, list): + groups[-1].append(_evaluate_markers(marker, environment)) + elif isinstance(marker, tuple): + lhs, op, rhs = marker + + if isinstance(lhs, Variable): + environment_key = lhs.value + lhs_value = environment[environment_key] + rhs_value = rhs.value + else: + lhs_value = lhs.value + environment_key = rhs.value + rhs_value = environment[environment_key] + assert isinstance(lhs_value, str), "lhs must be a string" + lhs_value, rhs_value = _normalize(lhs_value, rhs_value, key=environment_key) + groups[-1].append(_eval_op(lhs_value, op, rhs_value)) + else: + assert marker in ["and", "or"] + if marker == "or": + groups.append([]) + + return any(all(item) for item in groups) + + +def format_full_version(info: sys._version_info) -> str: + version = f"{info.major}.{info.minor}.{info.micro}" + kind = info.releaselevel + if kind != "final": + version += kind[0] + str(info.serial) + return version + + +def default_environment() -> Environment: + iver = format_full_version(sys.implementation.version) + implementation_name = sys.implementation.name + return { + "implementation_name": implementation_name, + "implementation_version": iver, + "os_name": os.name, + "platform_machine": platform.machine(), + "platform_release": platform.release(), + "platform_system": platform.system(), + "platform_version": platform.version(), + "python_full_version": platform.python_version(), + "platform_python_implementation": platform.python_implementation(), + "python_version": ".".join(platform.python_version_tuple()[:2]), + "sys_platform": sys.platform, + } + + +class Marker: + def __init__(self, marker: str) -> None: + # Note: We create a Marker object without calling this constructor in + # packaging.requirements.Requirement. If any additional logic is + # added here, make sure to mirror/adapt Requirement. + try: + self._markers = _normalize_extra_values(_parse_marker(marker)) + # The attribute `_markers` can be described in terms of a recursive type: + # MarkerList = List[Union[Tuple[Node, ...], str, MarkerList]] + # + # For example, the following expression: + # python_version > "3.6" or (python_version == "3.6" and os_name == "unix") + # + # is parsed into: + # [ + # (, ')>, ), + # 'and', + # [ + # (, , ), + # 'or', + # (, , ) + # ] + # ] + except ParserSyntaxError as e: + raise InvalidMarker(str(e)) from e + + def __str__(self) -> str: + return _format_marker(self._markers) + + def __repr__(self) -> str: + return f"" + + def __hash__(self) -> int: + return hash((self.__class__.__name__, str(self))) + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Marker): + return NotImplemented + + return str(self) == str(other) + + def evaluate( + self, + environment: dict[str, str] | None = None, + context: EvaluateContext = "metadata", + ) -> bool: + """Evaluate a marker. + + Return the boolean from evaluating the given marker against the + environment. environment is an optional argument to override all or + part of the determined environment. The *context* parameter specifies what + context the markers are being evaluated for, which influences what markers + are considered valid. Acceptable values are "metadata" (for core metadata; + default), "lock_file", and "requirement" (i.e. all other situations). + + The environment is determined from the current Python process. + """ + current_environment = cast( + "dict[str, str | AbstractSet[str]]", default_environment() + ) + if context == "lock_file": + current_environment.update( + extras=frozenset(), dependency_groups=frozenset() + ) + elif context == "metadata": + current_environment["extra"] = "" + if environment is not None: + current_environment.update(environment) + # The API used to allow setting extra to None. We need to handle this + # case for backwards compatibility. + if "extra" in current_environment and current_environment["extra"] is None: + current_environment["extra"] = "" + + return _evaluate_markers( + self._markers, _repair_python_full_version(current_environment) + ) + + +def _repair_python_full_version( + env: dict[str, str | AbstractSet[str]], +) -> dict[str, str | AbstractSet[str]]: + """ + Work around platform.python_version() returning something that is not PEP 440 + compliant for non-tagged Python builds. + """ + python_full_version = cast(str, env["python_full_version"]) + if python_full_version.endswith("+"): + env["python_full_version"] = f"{python_full_version}local" + return env diff --git a/lib/python3.1/site-packages/packaging/metadata.py b/lib/python3.1/site-packages/packaging/metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..3bd8602d36c53374f1f10a0716d28fdc64e2d3ac --- /dev/null +++ b/lib/python3.1/site-packages/packaging/metadata.py @@ -0,0 +1,862 @@ +from __future__ import annotations + +import email.feedparser +import email.header +import email.message +import email.parser +import email.policy +import pathlib +import sys +import typing +from typing import ( + Any, + Callable, + Generic, + Literal, + TypedDict, + cast, +) + +from . import licenses, requirements, specifiers, utils +from . import version as version_module +from .licenses import NormalizedLicenseExpression + +T = typing.TypeVar("T") + + +if sys.version_info >= (3, 11): # pragma: no cover + ExceptionGroup = ExceptionGroup +else: # pragma: no cover + + class ExceptionGroup(Exception): + """A minimal implementation of :external:exc:`ExceptionGroup` from Python 3.11. + + If :external:exc:`ExceptionGroup` is already defined by Python itself, + that version is used instead. + """ + + message: str + exceptions: list[Exception] + + def __init__(self, message: str, exceptions: list[Exception]) -> None: + self.message = message + self.exceptions = exceptions + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.message!r}, {self.exceptions!r})" + + +class InvalidMetadata(ValueError): + """A metadata field contains invalid data.""" + + field: str + """The name of the field that contains invalid data.""" + + def __init__(self, field: str, message: str) -> None: + self.field = field + super().__init__(message) + + +# The RawMetadata class attempts to make as few assumptions about the underlying +# serialization formats as possible. The idea is that as long as a serialization +# formats offer some very basic primitives in *some* way then we can support +# serializing to and from that format. +class RawMetadata(TypedDict, total=False): + """A dictionary of raw core metadata. + + Each field in core metadata maps to a key of this dictionary (when data is + provided). The key is lower-case and underscores are used instead of dashes + compared to the equivalent core metadata field. Any core metadata field that + can be specified multiple times or can hold multiple values in a single + field have a key with a plural name. See :class:`Metadata` whose attributes + match the keys of this dictionary. + + Core metadata fields that can be specified multiple times are stored as a + list or dict depending on which is appropriate for the field. Any fields + which hold multiple values in a single field are stored as a list. + + """ + + # Metadata 1.0 - PEP 241 + metadata_version: str + name: str + version: str + platforms: list[str] + summary: str + description: str + keywords: list[str] + home_page: str + author: str + author_email: str + license: str + + # Metadata 1.1 - PEP 314 + supported_platforms: list[str] + download_url: str + classifiers: list[str] + requires: list[str] + provides: list[str] + obsoletes: list[str] + + # Metadata 1.2 - PEP 345 + maintainer: str + maintainer_email: str + requires_dist: list[str] + provides_dist: list[str] + obsoletes_dist: list[str] + requires_python: str + requires_external: list[str] + project_urls: dict[str, str] + + # Metadata 2.0 + # PEP 426 attempted to completely revamp the metadata format + # but got stuck without ever being able to build consensus on + # it and ultimately ended up withdrawn. + # + # However, a number of tools had started emitting METADATA with + # `2.0` Metadata-Version, so for historical reasons, this version + # was skipped. + + # Metadata 2.1 - PEP 566 + description_content_type: str + provides_extra: list[str] + + # Metadata 2.2 - PEP 643 + dynamic: list[str] + + # Metadata 2.3 - PEP 685 + # No new fields were added in PEP 685, just some edge case were + # tightened up to provide better interoptability. + + # Metadata 2.4 - PEP 639 + license_expression: str + license_files: list[str] + + +_STRING_FIELDS = { + "author", + "author_email", + "description", + "description_content_type", + "download_url", + "home_page", + "license", + "license_expression", + "maintainer", + "maintainer_email", + "metadata_version", + "name", + "requires_python", + "summary", + "version", +} + +_LIST_FIELDS = { + "classifiers", + "dynamic", + "license_files", + "obsoletes", + "obsoletes_dist", + "platforms", + "provides", + "provides_dist", + "provides_extra", + "requires", + "requires_dist", + "requires_external", + "supported_platforms", +} + +_DICT_FIELDS = { + "project_urls", +} + + +def _parse_keywords(data: str) -> list[str]: + """Split a string of comma-separated keywords into a list of keywords.""" + return [k.strip() for k in data.split(",")] + + +def _parse_project_urls(data: list[str]) -> dict[str, str]: + """Parse a list of label/URL string pairings separated by a comma.""" + urls = {} + for pair in data: + # Our logic is slightly tricky here as we want to try and do + # *something* reasonable with malformed data. + # + # The main thing that we have to worry about, is data that does + # not have a ',' at all to split the label from the Value. There + # isn't a singular right answer here, and we will fail validation + # later on (if the caller is validating) so it doesn't *really* + # matter, but since the missing value has to be an empty str + # and our return value is dict[str, str], if we let the key + # be the missing value, then they'd have multiple '' values that + # overwrite each other in a accumulating dict. + # + # The other potentional issue is that it's possible to have the + # same label multiple times in the metadata, with no solid "right" + # answer with what to do in that case. As such, we'll do the only + # thing we can, which is treat the field as unparseable and add it + # to our list of unparsed fields. + parts = [p.strip() for p in pair.split(",", 1)] + parts.extend([""] * (max(0, 2 - len(parts)))) # Ensure 2 items + + # TODO: The spec doesn't say anything about if the keys should be + # considered case sensitive or not... logically they should + # be case-preserving and case-insensitive, but doing that + # would open up more cases where we might have duplicate + # entries. + label, url = parts + if label in urls: + # The label already exists in our set of urls, so this field + # is unparseable, and we can just add the whole thing to our + # unparseable data and stop processing it. + raise KeyError("duplicate labels in project urls") + urls[label] = url + + return urls + + +def _get_payload(msg: email.message.Message, source: bytes | str) -> str: + """Get the body of the message.""" + # If our source is a str, then our caller has managed encodings for us, + # and we don't need to deal with it. + if isinstance(source, str): + payload = msg.get_payload() + assert isinstance(payload, str) + return payload + # If our source is a bytes, then we're managing the encoding and we need + # to deal with it. + else: + bpayload = msg.get_payload(decode=True) + assert isinstance(bpayload, bytes) + try: + return bpayload.decode("utf8", "strict") + except UnicodeDecodeError as exc: + raise ValueError("payload in an invalid encoding") from exc + + +# The various parse_FORMAT functions here are intended to be as lenient as +# possible in their parsing, while still returning a correctly typed +# RawMetadata. +# +# To aid in this, we also generally want to do as little touching of the +# data as possible, except where there are possibly some historic holdovers +# that make valid data awkward to work with. +# +# While this is a lower level, intermediate format than our ``Metadata`` +# class, some light touch ups can make a massive difference in usability. + +# Map METADATA fields to RawMetadata. +_EMAIL_TO_RAW_MAPPING = { + "author": "author", + "author-email": "author_email", + "classifier": "classifiers", + "description": "description", + "description-content-type": "description_content_type", + "download-url": "download_url", + "dynamic": "dynamic", + "home-page": "home_page", + "keywords": "keywords", + "license": "license", + "license-expression": "license_expression", + "license-file": "license_files", + "maintainer": "maintainer", + "maintainer-email": "maintainer_email", + "metadata-version": "metadata_version", + "name": "name", + "obsoletes": "obsoletes", + "obsoletes-dist": "obsoletes_dist", + "platform": "platforms", + "project-url": "project_urls", + "provides": "provides", + "provides-dist": "provides_dist", + "provides-extra": "provides_extra", + "requires": "requires", + "requires-dist": "requires_dist", + "requires-external": "requires_external", + "requires-python": "requires_python", + "summary": "summary", + "supported-platform": "supported_platforms", + "version": "version", +} +_RAW_TO_EMAIL_MAPPING = {raw: email for email, raw in _EMAIL_TO_RAW_MAPPING.items()} + + +def parse_email(data: bytes | str) -> tuple[RawMetadata, dict[str, list[str]]]: + """Parse a distribution's metadata stored as email headers (e.g. from ``METADATA``). + + This function returns a two-item tuple of dicts. The first dict is of + recognized fields from the core metadata specification. Fields that can be + parsed and translated into Python's built-in types are converted + appropriately. All other fields are left as-is. Fields that are allowed to + appear multiple times are stored as lists. + + The second dict contains all other fields from the metadata. This includes + any unrecognized fields. It also includes any fields which are expected to + be parsed into a built-in type but were not formatted appropriately. Finally, + any fields that are expected to appear only once but are repeated are + included in this dict. + + """ + raw: dict[str, str | list[str] | dict[str, str]] = {} + unparsed: dict[str, list[str]] = {} + + if isinstance(data, str): + parsed = email.parser.Parser(policy=email.policy.compat32).parsestr(data) + else: + parsed = email.parser.BytesParser(policy=email.policy.compat32).parsebytes(data) + + # We have to wrap parsed.keys() in a set, because in the case of multiple + # values for a key (a list), the key will appear multiple times in the + # list of keys, but we're avoiding that by using get_all(). + for name in frozenset(parsed.keys()): + # Header names in RFC are case insensitive, so we'll normalize to all + # lower case to make comparisons easier. + name = name.lower() + + # We use get_all() here, even for fields that aren't multiple use, + # because otherwise someone could have e.g. two Name fields, and we + # would just silently ignore it rather than doing something about it. + headers = parsed.get_all(name) or [] + + # The way the email module works when parsing bytes is that it + # unconditionally decodes the bytes as ascii using the surrogateescape + # handler. When you pull that data back out (such as with get_all() ), + # it looks to see if the str has any surrogate escapes, and if it does + # it wraps it in a Header object instead of returning the string. + # + # As such, we'll look for those Header objects, and fix up the encoding. + value = [] + # Flag if we have run into any issues processing the headers, thus + # signalling that the data belongs in 'unparsed'. + valid_encoding = True + for h in headers: + # It's unclear if this can return more types than just a Header or + # a str, so we'll just assert here to make sure. + assert isinstance(h, (email.header.Header, str)) + + # If it's a header object, we need to do our little dance to get + # the real data out of it. In cases where there is invalid data + # we're going to end up with mojibake, but there's no obvious, good + # way around that without reimplementing parts of the Header object + # ourselves. + # + # That should be fine since, if mojibacked happens, this key is + # going into the unparsed dict anyways. + if isinstance(h, email.header.Header): + # The Header object stores it's data as chunks, and each chunk + # can be independently encoded, so we'll need to check each + # of them. + chunks: list[tuple[bytes, str | None]] = [] + for bin, encoding in email.header.decode_header(h): + try: + bin.decode("utf8", "strict") + except UnicodeDecodeError: + # Enable mojibake. + encoding = "latin1" + valid_encoding = False + else: + encoding = "utf8" + chunks.append((bin, encoding)) + + # Turn our chunks back into a Header object, then let that + # Header object do the right thing to turn them into a + # string for us. + value.append(str(email.header.make_header(chunks))) + # This is already a string, so just add it. + else: + value.append(h) + + # We've processed all of our values to get them into a list of str, + # but we may have mojibake data, in which case this is an unparsed + # field. + if not valid_encoding: + unparsed[name] = value + continue + + raw_name = _EMAIL_TO_RAW_MAPPING.get(name) + if raw_name is None: + # This is a bit of a weird situation, we've encountered a key that + # we don't know what it means, so we don't know whether it's meant + # to be a list or not. + # + # Since we can't really tell one way or another, we'll just leave it + # as a list, even though it may be a single item list, because that's + # what makes the most sense for email headers. + unparsed[name] = value + continue + + # If this is one of our string fields, then we'll check to see if our + # value is a list of a single item. If it is then we'll assume that + # it was emitted as a single string, and unwrap the str from inside + # the list. + # + # If it's any other kind of data, then we haven't the faintest clue + # what we should parse it as, and we have to just add it to our list + # of unparsed stuff. + if raw_name in _STRING_FIELDS and len(value) == 1: + raw[raw_name] = value[0] + # If this is one of our list of string fields, then we can just assign + # the value, since email *only* has strings, and our get_all() call + # above ensures that this is a list. + elif raw_name in _LIST_FIELDS: + raw[raw_name] = value + # Special Case: Keywords + # The keywords field is implemented in the metadata spec as a str, + # but it conceptually is a list of strings, and is serialized using + # ", ".join(keywords), so we'll do some light data massaging to turn + # this into what it logically is. + elif raw_name == "keywords" and len(value) == 1: + raw[raw_name] = _parse_keywords(value[0]) + # Special Case: Project-URL + # The project urls is implemented in the metadata spec as a list of + # specially-formatted strings that represent a key and a value, which + # is fundamentally a mapping, however the email format doesn't support + # mappings in a sane way, so it was crammed into a list of strings + # instead. + # + # We will do a little light data massaging to turn this into a map as + # it logically should be. + elif raw_name == "project_urls": + try: + raw[raw_name] = _parse_project_urls(value) + except KeyError: + unparsed[name] = value + # Nothing that we've done has managed to parse this, so it'll just + # throw it in our unparseable data and move on. + else: + unparsed[name] = value + + # We need to support getting the Description from the message payload in + # addition to getting it from the the headers. This does mean, though, there + # is the possibility of it being set both ways, in which case we put both + # in 'unparsed' since we don't know which is right. + try: + payload = _get_payload(parsed, data) + except ValueError: + unparsed.setdefault("description", []).append( + parsed.get_payload(decode=isinstance(data, bytes)) # type: ignore[call-overload] + ) + else: + if payload: + # Check to see if we've already got a description, if so then both + # it, and this body move to unparseable. + if "description" in raw: + description_header = cast(str, raw.pop("description")) + unparsed.setdefault("description", []).extend( + [description_header, payload] + ) + elif "description" in unparsed: + unparsed["description"].append(payload) + else: + raw["description"] = payload + + # We need to cast our `raw` to a metadata, because a TypedDict only support + # literal key names, but we're computing our key names on purpose, but the + # way this function is implemented, our `TypedDict` can only have valid key + # names. + return cast(RawMetadata, raw), unparsed + + +_NOT_FOUND = object() + + +# Keep the two values in sync. +_VALID_METADATA_VERSIONS = ["1.0", "1.1", "1.2", "2.1", "2.2", "2.3", "2.4"] +_MetadataVersion = Literal["1.0", "1.1", "1.2", "2.1", "2.2", "2.3", "2.4"] + +_REQUIRED_ATTRS = frozenset(["metadata_version", "name", "version"]) + + +class _Validator(Generic[T]): + """Validate a metadata field. + + All _process_*() methods correspond to a core metadata field. The method is + called with the field's raw value. If the raw value is valid it is returned + in its "enriched" form (e.g. ``version.Version`` for the ``Version`` field). + If the raw value is invalid, :exc:`InvalidMetadata` is raised (with a cause + as appropriate). + """ + + name: str + raw_name: str + added: _MetadataVersion + + def __init__( + self, + *, + added: _MetadataVersion = "1.0", + ) -> None: + self.added = added + + def __set_name__(self, _owner: Metadata, name: str) -> None: + self.name = name + self.raw_name = _RAW_TO_EMAIL_MAPPING[name] + + def __get__(self, instance: Metadata, _owner: type[Metadata]) -> T: + # With Python 3.8, the caching can be replaced with functools.cached_property(). + # No need to check the cache as attribute lookup will resolve into the + # instance's __dict__ before __get__ is called. + cache = instance.__dict__ + value = instance._raw.get(self.name) + + # To make the _process_* methods easier, we'll check if the value is None + # and if this field is NOT a required attribute, and if both of those + # things are true, we'll skip the the converter. This will mean that the + # converters never have to deal with the None union. + if self.name in _REQUIRED_ATTRS or value is not None: + try: + converter: Callable[[Any], T] = getattr(self, f"_process_{self.name}") + except AttributeError: + pass + else: + value = converter(value) + + cache[self.name] = value + try: + del instance._raw[self.name] # type: ignore[misc] + except KeyError: + pass + + return cast(T, value) + + def _invalid_metadata( + self, msg: str, cause: Exception | None = None + ) -> InvalidMetadata: + exc = InvalidMetadata( + self.raw_name, msg.format_map({"field": repr(self.raw_name)}) + ) + exc.__cause__ = cause + return exc + + def _process_metadata_version(self, value: str) -> _MetadataVersion: + # Implicitly makes Metadata-Version required. + if value not in _VALID_METADATA_VERSIONS: + raise self._invalid_metadata(f"{value!r} is not a valid metadata version") + return cast(_MetadataVersion, value) + + def _process_name(self, value: str) -> str: + if not value: + raise self._invalid_metadata("{field} is a required field") + # Validate the name as a side-effect. + try: + utils.canonicalize_name(value, validate=True) + except utils.InvalidName as exc: + raise self._invalid_metadata( + f"{value!r} is invalid for {{field}}", cause=exc + ) from exc + else: + return value + + def _process_version(self, value: str) -> version_module.Version: + if not value: + raise self._invalid_metadata("{field} is a required field") + try: + return version_module.parse(value) + except version_module.InvalidVersion as exc: + raise self._invalid_metadata( + f"{value!r} is invalid for {{field}}", cause=exc + ) from exc + + def _process_summary(self, value: str) -> str: + """Check the field contains no newlines.""" + if "\n" in value: + raise self._invalid_metadata("{field} must be a single line") + return value + + def _process_description_content_type(self, value: str) -> str: + content_types = {"text/plain", "text/x-rst", "text/markdown"} + message = email.message.EmailMessage() + message["content-type"] = value + + content_type, parameters = ( + # Defaults to `text/plain` if parsing failed. + message.get_content_type().lower(), + message["content-type"].params, + ) + # Check if content-type is valid or defaulted to `text/plain` and thus was + # not parseable. + if content_type not in content_types or content_type not in value.lower(): + raise self._invalid_metadata( + f"{{field}} must be one of {list(content_types)}, not {value!r}" + ) + + charset = parameters.get("charset", "UTF-8") + if charset != "UTF-8": + raise self._invalid_metadata( + f"{{field}} can only specify the UTF-8 charset, not {list(charset)}" + ) + + markdown_variants = {"GFM", "CommonMark"} + variant = parameters.get("variant", "GFM") # Use an acceptable default. + if content_type == "text/markdown" and variant not in markdown_variants: + raise self._invalid_metadata( + f"valid Markdown variants for {{field}} are {list(markdown_variants)}, " + f"not {variant!r}", + ) + return value + + def _process_dynamic(self, value: list[str]) -> list[str]: + for dynamic_field in map(str.lower, value): + if dynamic_field in {"name", "version", "metadata-version"}: + raise self._invalid_metadata( + f"{dynamic_field!r} is not allowed as a dynamic field" + ) + elif dynamic_field not in _EMAIL_TO_RAW_MAPPING: + raise self._invalid_metadata( + f"{dynamic_field!r} is not a valid dynamic field" + ) + return list(map(str.lower, value)) + + def _process_provides_extra( + self, + value: list[str], + ) -> list[utils.NormalizedName]: + normalized_names = [] + try: + for name in value: + normalized_names.append(utils.canonicalize_name(name, validate=True)) + except utils.InvalidName as exc: + raise self._invalid_metadata( + f"{name!r} is invalid for {{field}}", cause=exc + ) from exc + else: + return normalized_names + + def _process_requires_python(self, value: str) -> specifiers.SpecifierSet: + try: + return specifiers.SpecifierSet(value) + except specifiers.InvalidSpecifier as exc: + raise self._invalid_metadata( + f"{value!r} is invalid for {{field}}", cause=exc + ) from exc + + def _process_requires_dist( + self, + value: list[str], + ) -> list[requirements.Requirement]: + reqs = [] + try: + for req in value: + reqs.append(requirements.Requirement(req)) + except requirements.InvalidRequirement as exc: + raise self._invalid_metadata( + f"{req!r} is invalid for {{field}}", cause=exc + ) from exc + else: + return reqs + + def _process_license_expression( + self, value: str + ) -> NormalizedLicenseExpression | None: + try: + return licenses.canonicalize_license_expression(value) + except ValueError as exc: + raise self._invalid_metadata( + f"{value!r} is invalid for {{field}}", cause=exc + ) from exc + + def _process_license_files(self, value: list[str]) -> list[str]: + paths = [] + for path in value: + if ".." in path: + raise self._invalid_metadata( + f"{path!r} is invalid for {{field}}, " + "parent directory indicators are not allowed" + ) + if "*" in path: + raise self._invalid_metadata( + f"{path!r} is invalid for {{field}}, paths must be resolved" + ) + if ( + pathlib.PurePosixPath(path).is_absolute() + or pathlib.PureWindowsPath(path).is_absolute() + ): + raise self._invalid_metadata( + f"{path!r} is invalid for {{field}}, paths must be relative" + ) + if pathlib.PureWindowsPath(path).as_posix() != path: + raise self._invalid_metadata( + f"{path!r} is invalid for {{field}}, paths must use '/' delimiter" + ) + paths.append(path) + return paths + + +class Metadata: + """Representation of distribution metadata. + + Compared to :class:`RawMetadata`, this class provides objects representing + metadata fields instead of only using built-in types. Any invalid metadata + will cause :exc:`InvalidMetadata` to be raised (with a + :py:attr:`~BaseException.__cause__` attribute as appropriate). + """ + + _raw: RawMetadata + + @classmethod + def from_raw(cls, data: RawMetadata, *, validate: bool = True) -> Metadata: + """Create an instance from :class:`RawMetadata`. + + If *validate* is true, all metadata will be validated. All exceptions + related to validation will be gathered and raised as an :class:`ExceptionGroup`. + """ + ins = cls() + ins._raw = data.copy() # Mutations occur due to caching enriched values. + + if validate: + exceptions: list[Exception] = [] + try: + metadata_version = ins.metadata_version + metadata_age = _VALID_METADATA_VERSIONS.index(metadata_version) + except InvalidMetadata as metadata_version_exc: + exceptions.append(metadata_version_exc) + metadata_version = None + + # Make sure to check for the fields that are present, the required + # fields (so their absence can be reported). + fields_to_check = frozenset(ins._raw) | _REQUIRED_ATTRS + # Remove fields that have already been checked. + fields_to_check -= {"metadata_version"} + + for key in fields_to_check: + try: + if metadata_version: + # Can't use getattr() as that triggers descriptor protocol which + # will fail due to no value for the instance argument. + try: + field_metadata_version = cls.__dict__[key].added + except KeyError: + exc = InvalidMetadata(key, f"unrecognized field: {key!r}") + exceptions.append(exc) + continue + field_age = _VALID_METADATA_VERSIONS.index( + field_metadata_version + ) + if field_age > metadata_age: + field = _RAW_TO_EMAIL_MAPPING[key] + exc = InvalidMetadata( + field, + f"{field} introduced in metadata version " + f"{field_metadata_version}, not {metadata_version}", + ) + exceptions.append(exc) + continue + getattr(ins, key) + except InvalidMetadata as exc: + exceptions.append(exc) + + if exceptions: + raise ExceptionGroup("invalid metadata", exceptions) + + return ins + + @classmethod + def from_email(cls, data: bytes | str, *, validate: bool = True) -> Metadata: + """Parse metadata from email headers. + + If *validate* is true, the metadata will be validated. All exceptions + related to validation will be gathered and raised as an :class:`ExceptionGroup`. + """ + raw, unparsed = parse_email(data) + + if validate: + exceptions: list[Exception] = [] + for unparsed_key in unparsed: + if unparsed_key in _EMAIL_TO_RAW_MAPPING: + message = f"{unparsed_key!r} has invalid data" + else: + message = f"unrecognized field: {unparsed_key!r}" + exceptions.append(InvalidMetadata(unparsed_key, message)) + + if exceptions: + raise ExceptionGroup("unparsed", exceptions) + + try: + return cls.from_raw(raw, validate=validate) + except ExceptionGroup as exc_group: + raise ExceptionGroup( + "invalid or unparsed metadata", exc_group.exceptions + ) from None + + metadata_version: _Validator[_MetadataVersion] = _Validator() + """:external:ref:`core-metadata-metadata-version` + (required; validated to be a valid metadata version)""" + # `name` is not normalized/typed to NormalizedName so as to provide access to + # the original/raw name. + name: _Validator[str] = _Validator() + """:external:ref:`core-metadata-name` + (required; validated using :func:`~packaging.utils.canonicalize_name` and its + *validate* parameter)""" + version: _Validator[version_module.Version] = _Validator() + """:external:ref:`core-metadata-version` (required)""" + dynamic: _Validator[list[str] | None] = _Validator( + added="2.2", + ) + """:external:ref:`core-metadata-dynamic` + (validated against core metadata field names and lowercased)""" + platforms: _Validator[list[str] | None] = _Validator() + """:external:ref:`core-metadata-platform`""" + supported_platforms: _Validator[list[str] | None] = _Validator(added="1.1") + """:external:ref:`core-metadata-supported-platform`""" + summary: _Validator[str | None] = _Validator() + """:external:ref:`core-metadata-summary` (validated to contain no newlines)""" + description: _Validator[str | None] = _Validator() # TODO 2.1: can be in body + """:external:ref:`core-metadata-description`""" + description_content_type: _Validator[str | None] = _Validator(added="2.1") + """:external:ref:`core-metadata-description-content-type` (validated)""" + keywords: _Validator[list[str] | None] = _Validator() + """:external:ref:`core-metadata-keywords`""" + home_page: _Validator[str | None] = _Validator() + """:external:ref:`core-metadata-home-page`""" + download_url: _Validator[str | None] = _Validator(added="1.1") + """:external:ref:`core-metadata-download-url`""" + author: _Validator[str | None] = _Validator() + """:external:ref:`core-metadata-author`""" + author_email: _Validator[str | None] = _Validator() + """:external:ref:`core-metadata-author-email`""" + maintainer: _Validator[str | None] = _Validator(added="1.2") + """:external:ref:`core-metadata-maintainer`""" + maintainer_email: _Validator[str | None] = _Validator(added="1.2") + """:external:ref:`core-metadata-maintainer-email`""" + license: _Validator[str | None] = _Validator() + """:external:ref:`core-metadata-license`""" + license_expression: _Validator[NormalizedLicenseExpression | None] = _Validator( + added="2.4" + ) + """:external:ref:`core-metadata-license-expression`""" + license_files: _Validator[list[str] | None] = _Validator(added="2.4") + """:external:ref:`core-metadata-license-file`""" + classifiers: _Validator[list[str] | None] = _Validator(added="1.1") + """:external:ref:`core-metadata-classifier`""" + requires_dist: _Validator[list[requirements.Requirement] | None] = _Validator( + added="1.2" + ) + """:external:ref:`core-metadata-requires-dist`""" + requires_python: _Validator[specifiers.SpecifierSet | None] = _Validator( + added="1.2" + ) + """:external:ref:`core-metadata-requires-python`""" + # Because `Requires-External` allows for non-PEP 440 version specifiers, we + # don't do any processing on the values. + requires_external: _Validator[list[str] | None] = _Validator(added="1.2") + """:external:ref:`core-metadata-requires-external`""" + project_urls: _Validator[dict[str, str] | None] = _Validator(added="1.2") + """:external:ref:`core-metadata-project-url`""" + # PEP 685 lets us raise an error if an extra doesn't pass `Name` validation + # regardless of metadata version. + provides_extra: _Validator[list[utils.NormalizedName] | None] = _Validator( + added="2.1", + ) + """:external:ref:`core-metadata-provides-extra`""" + provides_dist: _Validator[list[str] | None] = _Validator(added="1.2") + """:external:ref:`core-metadata-provides-dist`""" + obsoletes_dist: _Validator[list[str] | None] = _Validator(added="1.2") + """:external:ref:`core-metadata-obsoletes-dist`""" + requires: _Validator[list[str] | None] = _Validator(added="1.1") + """``Requires`` (deprecated)""" + provides: _Validator[list[str] | None] = _Validator(added="1.1") + """``Provides`` (deprecated)""" + obsoletes: _Validator[list[str] | None] = _Validator(added="1.1") + """``Obsoletes`` (deprecated)""" diff --git a/lib/python3.1/site-packages/packaging/py.typed b/lib/python3.1/site-packages/packaging/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.1/site-packages/packaging/requirements.py b/lib/python3.1/site-packages/packaging/requirements.py new file mode 100644 index 0000000000000000000000000000000000000000..4e068c9567def3564f238a76fe7ab46b569f33e5 --- /dev/null +++ b/lib/python3.1/site-packages/packaging/requirements.py @@ -0,0 +1,91 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import annotations + +from typing import Any, Iterator + +from ._parser import parse_requirement as _parse_requirement +from ._tokenizer import ParserSyntaxError +from .markers import Marker, _normalize_extra_values +from .specifiers import SpecifierSet +from .utils import canonicalize_name + + +class InvalidRequirement(ValueError): + """ + An invalid requirement was found, users should refer to PEP 508. + """ + + +class Requirement: + """Parse a requirement. + + Parse a given requirement string into its parts, such as name, specifier, + URL, and extras. Raises InvalidRequirement on a badly-formed requirement + string. + """ + + # TODO: Can we test whether something is contained within a requirement? + # If so how do we do that? Do we need to test against the _name_ of + # the thing as well as the version? What about the markers? + # TODO: Can we normalize the name and extra name? + + def __init__(self, requirement_string: str) -> None: + try: + parsed = _parse_requirement(requirement_string) + except ParserSyntaxError as e: + raise InvalidRequirement(str(e)) from e + + self.name: str = parsed.name + self.url: str | None = parsed.url or None + self.extras: set[str] = set(parsed.extras or []) + self.specifier: SpecifierSet = SpecifierSet(parsed.specifier) + self.marker: Marker | None = None + if parsed.marker is not None: + self.marker = Marker.__new__(Marker) + self.marker._markers = _normalize_extra_values(parsed.marker) + + def _iter_parts(self, name: str) -> Iterator[str]: + yield name + + if self.extras: + formatted_extras = ",".join(sorted(self.extras)) + yield f"[{formatted_extras}]" + + if self.specifier: + yield str(self.specifier) + + if self.url: + yield f"@ {self.url}" + if self.marker: + yield " " + + if self.marker: + yield f"; {self.marker}" + + def __str__(self) -> str: + return "".join(self._iter_parts(self.name)) + + def __repr__(self) -> str: + return f"" + + def __hash__(self) -> int: + return hash( + ( + self.__class__.__name__, + *self._iter_parts(canonicalize_name(self.name)), + ) + ) + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Requirement): + return NotImplemented + + return ( + canonicalize_name(self.name) == canonicalize_name(other.name) + and self.extras == other.extras + and self.specifier == other.specifier + and self.url == other.url + and self.marker == other.marker + ) diff --git a/lib/python3.1/site-packages/packaging/specifiers.py b/lib/python3.1/site-packages/packaging/specifiers.py new file mode 100644 index 0000000000000000000000000000000000000000..c8448043006f1c60ee92df69da47986d10229b1d --- /dev/null +++ b/lib/python3.1/site-packages/packaging/specifiers.py @@ -0,0 +1,1019 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +""" +.. testsetup:: + + from packaging.specifiers import Specifier, SpecifierSet, InvalidSpecifier + from packaging.version import Version +""" + +from __future__ import annotations + +import abc +import itertools +import re +from typing import Callable, Iterable, Iterator, TypeVar, Union + +from .utils import canonicalize_version +from .version import Version + +UnparsedVersion = Union[Version, str] +UnparsedVersionVar = TypeVar("UnparsedVersionVar", bound=UnparsedVersion) +CallableOperator = Callable[[Version, str], bool] + + +def _coerce_version(version: UnparsedVersion) -> Version: + if not isinstance(version, Version): + version = Version(version) + return version + + +class InvalidSpecifier(ValueError): + """ + Raised when attempting to create a :class:`Specifier` with a specifier + string that is invalid. + + >>> Specifier("lolwat") + Traceback (most recent call last): + ... + packaging.specifiers.InvalidSpecifier: Invalid specifier: 'lolwat' + """ + + +class BaseSpecifier(metaclass=abc.ABCMeta): + @abc.abstractmethod + def __str__(self) -> str: + """ + Returns the str representation of this Specifier-like object. This + should be representative of the Specifier itself. + """ + + @abc.abstractmethod + def __hash__(self) -> int: + """ + Returns a hash value for this Specifier-like object. + """ + + @abc.abstractmethod + def __eq__(self, other: object) -> bool: + """ + Returns a boolean representing whether or not the two Specifier-like + objects are equal. + + :param other: The other object to check against. + """ + + @property + @abc.abstractmethod + def prereleases(self) -> bool | None: + """Whether or not pre-releases as a whole are allowed. + + This can be set to either ``True`` or ``False`` to explicitly enable or disable + prereleases or it can be set to ``None`` (the default) to use default semantics. + """ + + @prereleases.setter + def prereleases(self, value: bool) -> None: + """Setter for :attr:`prereleases`. + + :param value: The value to set. + """ + + @abc.abstractmethod + def contains(self, item: str, prereleases: bool | None = None) -> bool: + """ + Determines if the given item is contained within this specifier. + """ + + @abc.abstractmethod + def filter( + self, iterable: Iterable[UnparsedVersionVar], prereleases: bool | None = None + ) -> Iterator[UnparsedVersionVar]: + """ + Takes an iterable of items and filters them so that only items which + are contained within this specifier are allowed in it. + """ + + +class Specifier(BaseSpecifier): + """This class abstracts handling of version specifiers. + + .. tip:: + + It is generally not required to instantiate this manually. You should instead + prefer to work with :class:`SpecifierSet` instead, which can parse + comma-separated version specifiers (which is what package metadata contains). + """ + + _operator_regex_str = r""" + (?P(~=|==|!=|<=|>=|<|>|===)) + """ + _version_regex_str = r""" + (?P + (?: + # The identity operators allow for an escape hatch that will + # do an exact string match of the version you wish to install. + # This will not be parsed by PEP 440 and we cannot determine + # any semantic meaning from it. This operator is discouraged + # but included entirely as an escape hatch. + (?<====) # Only match for the identity operator + \s* + [^\s;)]* # The arbitrary version can be just about anything, + # we match everything except for whitespace, a + # semi-colon for marker support, and a closing paren + # since versions can be enclosed in them. + ) + | + (?: + # The (non)equality operators allow for wild card and local + # versions to be specified so we have to define these two + # operators separately to enable that. + (?<===|!=) # Only match for equals and not equals + + \s* + v? + (?:[0-9]+!)? # epoch + [0-9]+(?:\.[0-9]+)* # release + + # You cannot use a wild card and a pre-release, post-release, a dev or + # local version together so group them with a | and make them optional. + (?: + \.\* # Wild card syntax of .* + | + (?: # pre release + [-_\.]? + (alpha|beta|preview|pre|a|b|c|rc) + [-_\.]? + [0-9]* + )? + (?: # post release + (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) + )? + (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + (?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local + )? + ) + | + (?: + # The compatible operator requires at least two digits in the + # release segment. + (?<=~=) # Only match for the compatible operator + + \s* + v? + (?:[0-9]+!)? # epoch + [0-9]+(?:\.[0-9]+)+ # release (We have a + instead of a *) + (?: # pre release + [-_\.]? + (alpha|beta|preview|pre|a|b|c|rc) + [-_\.]? + [0-9]* + )? + (?: # post release + (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) + )? + (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + ) + | + (?: + # All other operators only allow a sub set of what the + # (non)equality operators do. Specifically they do not allow + # local versions to be specified nor do they allow the prefix + # matching wild cards. + (?=": "greater_than_equal", + "<": "less_than", + ">": "greater_than", + "===": "arbitrary", + } + + def __init__(self, spec: str = "", prereleases: bool | None = None) -> None: + """Initialize a Specifier instance. + + :param spec: + The string representation of a specifier which will be parsed and + normalized before use. + :param prereleases: + This tells the specifier if it should accept prerelease versions if + applicable or not. The default of ``None`` will autodetect it from the + given specifiers. + :raises InvalidSpecifier: + If the given specifier is invalid (i.e. bad syntax). + """ + match = self._regex.search(spec) + if not match: + raise InvalidSpecifier(f"Invalid specifier: {spec!r}") + + self._spec: tuple[str, str] = ( + match.group("operator").strip(), + match.group("version").strip(), + ) + + # Store whether or not this Specifier should accept prereleases + self._prereleases = prereleases + + # https://github.com/python/mypy/pull/13475#pullrequestreview-1079784515 + @property # type: ignore[override] + def prereleases(self) -> bool: + # If there is an explicit prereleases set for this, then we'll just + # blindly use that. + if self._prereleases is not None: + return self._prereleases + + # Look at all of our specifiers and determine if they are inclusive + # operators, and if they are if they are including an explicit + # prerelease. + operator, version = self._spec + if operator in ["==", ">=", "<=", "~=", "===", ">", "<"]: + # The == specifier can include a trailing .*, if it does we + # want to remove before parsing. + if operator == "==" and version.endswith(".*"): + version = version[:-2] + + # Parse the version, and if it is a pre-release than this + # specifier allows pre-releases. + if Version(version).is_prerelease: + return True + + return False + + @prereleases.setter + def prereleases(self, value: bool) -> None: + self._prereleases = value + + @property + def operator(self) -> str: + """The operator of this specifier. + + >>> Specifier("==1.2.3").operator + '==' + """ + return self._spec[0] + + @property + def version(self) -> str: + """The version of this specifier. + + >>> Specifier("==1.2.3").version + '1.2.3' + """ + return self._spec[1] + + def __repr__(self) -> str: + """A representation of the Specifier that shows all internal state. + + >>> Specifier('>=1.0.0') + =1.0.0')> + >>> Specifier('>=1.0.0', prereleases=False) + =1.0.0', prereleases=False)> + >>> Specifier('>=1.0.0', prereleases=True) + =1.0.0', prereleases=True)> + """ + pre = ( + f", prereleases={self.prereleases!r}" + if self._prereleases is not None + else "" + ) + + return f"<{self.__class__.__name__}({str(self)!r}{pre})>" + + def __str__(self) -> str: + """A string representation of the Specifier that can be round-tripped. + + >>> str(Specifier('>=1.0.0')) + '>=1.0.0' + >>> str(Specifier('>=1.0.0', prereleases=False)) + '>=1.0.0' + """ + return "{}{}".format(*self._spec) + + @property + def _canonical_spec(self) -> tuple[str, str]: + canonical_version = canonicalize_version( + self._spec[1], + strip_trailing_zero=(self._spec[0] != "~="), + ) + return self._spec[0], canonical_version + + def __hash__(self) -> int: + return hash(self._canonical_spec) + + def __eq__(self, other: object) -> bool: + """Whether or not the two Specifier-like objects are equal. + + :param other: The other object to check against. + + The value of :attr:`prereleases` is ignored. + + >>> Specifier("==1.2.3") == Specifier("== 1.2.3.0") + True + >>> (Specifier("==1.2.3", prereleases=False) == + ... Specifier("==1.2.3", prereleases=True)) + True + >>> Specifier("==1.2.3") == "==1.2.3" + True + >>> Specifier("==1.2.3") == Specifier("==1.2.4") + False + >>> Specifier("==1.2.3") == Specifier("~=1.2.3") + False + """ + if isinstance(other, str): + try: + other = self.__class__(str(other)) + except InvalidSpecifier: + return NotImplemented + elif not isinstance(other, self.__class__): + return NotImplemented + + return self._canonical_spec == other._canonical_spec + + def _get_operator(self, op: str) -> CallableOperator: + operator_callable: CallableOperator = getattr( + self, f"_compare_{self._operators[op]}" + ) + return operator_callable + + def _compare_compatible(self, prospective: Version, spec: str) -> bool: + # Compatible releases have an equivalent combination of >= and ==. That + # is that ~=2.2 is equivalent to >=2.2,==2.*. This allows us to + # implement this in terms of the other specifiers instead of + # implementing it ourselves. The only thing we need to do is construct + # the other specifiers. + + # We want everything but the last item in the version, but we want to + # ignore suffix segments. + prefix = _version_join( + list(itertools.takewhile(_is_not_suffix, _version_split(spec)))[:-1] + ) + + # Add the prefix notation to the end of our string + prefix += ".*" + + return self._get_operator(">=")(prospective, spec) and self._get_operator("==")( + prospective, prefix + ) + + def _compare_equal(self, prospective: Version, spec: str) -> bool: + # We need special logic to handle prefix matching + if spec.endswith(".*"): + # In the case of prefix matching we want to ignore local segment. + normalized_prospective = canonicalize_version( + prospective.public, strip_trailing_zero=False + ) + # Get the normalized version string ignoring the trailing .* + normalized_spec = canonicalize_version(spec[:-2], strip_trailing_zero=False) + # Split the spec out by bangs and dots, and pretend that there is + # an implicit dot in between a release segment and a pre-release segment. + split_spec = _version_split(normalized_spec) + + # Split the prospective version out by bangs and dots, and pretend + # that there is an implicit dot in between a release segment and + # a pre-release segment. + split_prospective = _version_split(normalized_prospective) + + # 0-pad the prospective version before shortening it to get the correct + # shortened version. + padded_prospective, _ = _pad_version(split_prospective, split_spec) + + # Shorten the prospective version to be the same length as the spec + # so that we can determine if the specifier is a prefix of the + # prospective version or not. + shortened_prospective = padded_prospective[: len(split_spec)] + + return shortened_prospective == split_spec + else: + # Convert our spec string into a Version + spec_version = Version(spec) + + # If the specifier does not have a local segment, then we want to + # act as if the prospective version also does not have a local + # segment. + if not spec_version.local: + prospective = Version(prospective.public) + + return prospective == spec_version + + def _compare_not_equal(self, prospective: Version, spec: str) -> bool: + return not self._compare_equal(prospective, spec) + + def _compare_less_than_equal(self, prospective: Version, spec: str) -> bool: + # NB: Local version identifiers are NOT permitted in the version + # specifier, so local version labels can be universally removed from + # the prospective version. + return Version(prospective.public) <= Version(spec) + + def _compare_greater_than_equal(self, prospective: Version, spec: str) -> bool: + # NB: Local version identifiers are NOT permitted in the version + # specifier, so local version labels can be universally removed from + # the prospective version. + return Version(prospective.public) >= Version(spec) + + def _compare_less_than(self, prospective: Version, spec_str: str) -> bool: + # Convert our spec to a Version instance, since we'll want to work with + # it as a version. + spec = Version(spec_str) + + # Check to see if the prospective version is less than the spec + # version. If it's not we can short circuit and just return False now + # instead of doing extra unneeded work. + if not prospective < spec: + return False + + # This special case is here so that, unless the specifier itself + # includes is a pre-release version, that we do not accept pre-release + # versions for the version mentioned in the specifier (e.g. <3.1 should + # not match 3.1.dev0, but should match 3.0.dev0). + if not spec.is_prerelease and prospective.is_prerelease: + if Version(prospective.base_version) == Version(spec.base_version): + return False + + # If we've gotten to here, it means that prospective version is both + # less than the spec version *and* it's not a pre-release of the same + # version in the spec. + return True + + def _compare_greater_than(self, prospective: Version, spec_str: str) -> bool: + # Convert our spec to a Version instance, since we'll want to work with + # it as a version. + spec = Version(spec_str) + + # Check to see if the prospective version is greater than the spec + # version. If it's not we can short circuit and just return False now + # instead of doing extra unneeded work. + if not prospective > spec: + return False + + # This special case is here so that, unless the specifier itself + # includes is a post-release version, that we do not accept + # post-release versions for the version mentioned in the specifier + # (e.g. >3.1 should not match 3.0.post0, but should match 3.2.post0). + if not spec.is_postrelease and prospective.is_postrelease: + if Version(prospective.base_version) == Version(spec.base_version): + return False + + # Ensure that we do not allow a local version of the version mentioned + # in the specifier, which is technically greater than, to match. + if prospective.local is not None: + if Version(prospective.base_version) == Version(spec.base_version): + return False + + # If we've gotten to here, it means that prospective version is both + # greater than the spec version *and* it's not a pre-release of the + # same version in the spec. + return True + + def _compare_arbitrary(self, prospective: Version, spec: str) -> bool: + return str(prospective).lower() == str(spec).lower() + + def __contains__(self, item: str | Version) -> bool: + """Return whether or not the item is contained in this specifier. + + :param item: The item to check for. + + This is used for the ``in`` operator and behaves the same as + :meth:`contains` with no ``prereleases`` argument passed. + + >>> "1.2.3" in Specifier(">=1.2.3") + True + >>> Version("1.2.3") in Specifier(">=1.2.3") + True + >>> "1.0.0" in Specifier(">=1.2.3") + False + >>> "1.3.0a1" in Specifier(">=1.2.3") + False + >>> "1.3.0a1" in Specifier(">=1.2.3", prereleases=True) + True + """ + return self.contains(item) + + def contains(self, item: UnparsedVersion, prereleases: bool | None = None) -> bool: + """Return whether or not the item is contained in this specifier. + + :param item: + The item to check for, which can be a version string or a + :class:`Version` instance. + :param prereleases: + Whether or not to match prereleases with this Specifier. If set to + ``None`` (the default), it uses :attr:`prereleases` to determine + whether or not prereleases are allowed. + + >>> Specifier(">=1.2.3").contains("1.2.3") + True + >>> Specifier(">=1.2.3").contains(Version("1.2.3")) + True + >>> Specifier(">=1.2.3").contains("1.0.0") + False + >>> Specifier(">=1.2.3").contains("1.3.0a1") + False + >>> Specifier(">=1.2.3", prereleases=True).contains("1.3.0a1") + True + >>> Specifier(">=1.2.3").contains("1.3.0a1", prereleases=True) + True + """ + + # Determine if prereleases are to be allowed or not. + if prereleases is None: + prereleases = self.prereleases + + # Normalize item to a Version, this allows us to have a shortcut for + # "2.0" in Specifier(">=2") + normalized_item = _coerce_version(item) + + # Determine if we should be supporting prereleases in this specifier + # or not, if we do not support prereleases than we can short circuit + # logic if this version is a prereleases. + if normalized_item.is_prerelease and not prereleases: + return False + + # Actually do the comparison to determine if this item is contained + # within this Specifier or not. + operator_callable: CallableOperator = self._get_operator(self.operator) + return operator_callable(normalized_item, self.version) + + def filter( + self, iterable: Iterable[UnparsedVersionVar], prereleases: bool | None = None + ) -> Iterator[UnparsedVersionVar]: + """Filter items in the given iterable, that match the specifier. + + :param iterable: + An iterable that can contain version strings and :class:`Version` instances. + The items in the iterable will be filtered according to the specifier. + :param prereleases: + Whether or not to allow prereleases in the returned iterator. If set to + ``None`` (the default), it will be intelligently decide whether to allow + prereleases or not (based on the :attr:`prereleases` attribute, and + whether the only versions matching are prereleases). + + This method is smarter than just ``filter(Specifier().contains, [...])`` + because it implements the rule from :pep:`440` that a prerelease item + SHOULD be accepted if no other versions match the given specifier. + + >>> list(Specifier(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) + ['1.3'] + >>> list(Specifier(">=1.2.3").filter(["1.2", "1.2.3", "1.3", Version("1.4")])) + ['1.2.3', '1.3', ] + >>> list(Specifier(">=1.2.3").filter(["1.2", "1.5a1"])) + ['1.5a1'] + >>> list(Specifier(">=1.2.3").filter(["1.3", "1.5a1"], prereleases=True)) + ['1.3', '1.5a1'] + >>> list(Specifier(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) + ['1.3', '1.5a1'] + """ + + yielded = False + found_prereleases = [] + + kw = {"prereleases": prereleases if prereleases is not None else True} + + # Attempt to iterate over all the values in the iterable and if any of + # them match, yield them. + for version in iterable: + parsed_version = _coerce_version(version) + + if self.contains(parsed_version, **kw): + # If our version is a prerelease, and we were not set to allow + # prereleases, then we'll store it for later in case nothing + # else matches this specifier. + if parsed_version.is_prerelease and not ( + prereleases or self.prereleases + ): + found_prereleases.append(version) + # Either this is not a prerelease, or we should have been + # accepting prereleases from the beginning. + else: + yielded = True + yield version + + # Now that we've iterated over everything, determine if we've yielded + # any values, and if we have not and we have any prereleases stored up + # then we will go ahead and yield the prereleases. + if not yielded and found_prereleases: + for version in found_prereleases: + yield version + + +_prefix_regex = re.compile(r"^([0-9]+)((?:a|b|c|rc)[0-9]+)$") + + +def _version_split(version: str) -> list[str]: + """Split version into components. + + The split components are intended for version comparison. The logic does + not attempt to retain the original version string, so joining the + components back with :func:`_version_join` may not produce the original + version string. + """ + result: list[str] = [] + + epoch, _, rest = version.rpartition("!") + result.append(epoch or "0") + + for item in rest.split("."): + match = _prefix_regex.search(item) + if match: + result.extend(match.groups()) + else: + result.append(item) + return result + + +def _version_join(components: list[str]) -> str: + """Join split version components into a version string. + + This function assumes the input came from :func:`_version_split`, where the + first component must be the epoch (either empty or numeric), and all other + components numeric. + """ + epoch, *rest = components + return f"{epoch}!{'.'.join(rest)}" + + +def _is_not_suffix(segment: str) -> bool: + return not any( + segment.startswith(prefix) for prefix in ("dev", "a", "b", "rc", "post") + ) + + +def _pad_version(left: list[str], right: list[str]) -> tuple[list[str], list[str]]: + left_split, right_split = [], [] + + # Get the release segment of our versions + left_split.append(list(itertools.takewhile(lambda x: x.isdigit(), left))) + right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) + + # Get the rest of our versions + left_split.append(left[len(left_split[0]) :]) + right_split.append(right[len(right_split[0]) :]) + + # Insert our padding + left_split.insert(1, ["0"] * max(0, len(right_split[0]) - len(left_split[0]))) + right_split.insert(1, ["0"] * max(0, len(left_split[0]) - len(right_split[0]))) + + return ( + list(itertools.chain.from_iterable(left_split)), + list(itertools.chain.from_iterable(right_split)), + ) + + +class SpecifierSet(BaseSpecifier): + """This class abstracts handling of a set of version specifiers. + + It can be passed a single specifier (``>=3.0``), a comma-separated list of + specifiers (``>=3.0,!=3.1``), or no specifier at all. + """ + + def __init__( + self, + specifiers: str | Iterable[Specifier] = "", + prereleases: bool | None = None, + ) -> None: + """Initialize a SpecifierSet instance. + + :param specifiers: + The string representation of a specifier or a comma-separated list of + specifiers which will be parsed and normalized before use. + May also be an iterable of ``Specifier`` instances, which will be used + as is. + :param prereleases: + This tells the SpecifierSet if it should accept prerelease versions if + applicable or not. The default of ``None`` will autodetect it from the + given specifiers. + + :raises InvalidSpecifier: + If the given ``specifiers`` are not parseable than this exception will be + raised. + """ + + if isinstance(specifiers, str): + # Split on `,` to break each individual specifier into its own item, and + # strip each item to remove leading/trailing whitespace. + split_specifiers = [s.strip() for s in specifiers.split(",") if s.strip()] + + # Make each individual specifier a Specifier and save in a frozen set + # for later. + self._specs = frozenset(map(Specifier, split_specifiers)) + else: + # Save the supplied specifiers in a frozen set. + self._specs = frozenset(specifiers) + + # Store our prereleases value so we can use it later to determine if + # we accept prereleases or not. + self._prereleases = prereleases + + @property + def prereleases(self) -> bool | None: + # If we have been given an explicit prerelease modifier, then we'll + # pass that through here. + if self._prereleases is not None: + return self._prereleases + + # If we don't have any specifiers, and we don't have a forced value, + # then we'll just return None since we don't know if this should have + # pre-releases or not. + if not self._specs: + return None + + # Otherwise we'll see if any of the given specifiers accept + # prereleases, if any of them do we'll return True, otherwise False. + return any(s.prereleases for s in self._specs) + + @prereleases.setter + def prereleases(self, value: bool) -> None: + self._prereleases = value + + def __repr__(self) -> str: + """A representation of the specifier set that shows all internal state. + + Note that the ordering of the individual specifiers within the set may not + match the input string. + + >>> SpecifierSet('>=1.0.0,!=2.0.0') + =1.0.0')> + >>> SpecifierSet('>=1.0.0,!=2.0.0', prereleases=False) + =1.0.0', prereleases=False)> + >>> SpecifierSet('>=1.0.0,!=2.0.0', prereleases=True) + =1.0.0', prereleases=True)> + """ + pre = ( + f", prereleases={self.prereleases!r}" + if self._prereleases is not None + else "" + ) + + return f"" + + def __str__(self) -> str: + """A string representation of the specifier set that can be round-tripped. + + Note that the ordering of the individual specifiers within the set may not + match the input string. + + >>> str(SpecifierSet(">=1.0.0,!=1.0.1")) + '!=1.0.1,>=1.0.0' + >>> str(SpecifierSet(">=1.0.0,!=1.0.1", prereleases=False)) + '!=1.0.1,>=1.0.0' + """ + return ",".join(sorted(str(s) for s in self._specs)) + + def __hash__(self) -> int: + return hash(self._specs) + + def __and__(self, other: SpecifierSet | str) -> SpecifierSet: + """Return a SpecifierSet which is a combination of the two sets. + + :param other: The other object to combine with. + + >>> SpecifierSet(">=1.0.0,!=1.0.1") & '<=2.0.0,!=2.0.1' + =1.0.0')> + >>> SpecifierSet(">=1.0.0,!=1.0.1") & SpecifierSet('<=2.0.0,!=2.0.1') + =1.0.0')> + """ + if isinstance(other, str): + other = SpecifierSet(other) + elif not isinstance(other, SpecifierSet): + return NotImplemented + + specifier = SpecifierSet() + specifier._specs = frozenset(self._specs | other._specs) + + if self._prereleases is None and other._prereleases is not None: + specifier._prereleases = other._prereleases + elif self._prereleases is not None and other._prereleases is None: + specifier._prereleases = self._prereleases + elif self._prereleases == other._prereleases: + specifier._prereleases = self._prereleases + else: + raise ValueError( + "Cannot combine SpecifierSets with True and False prerelease overrides." + ) + + return specifier + + def __eq__(self, other: object) -> bool: + """Whether or not the two SpecifierSet-like objects are equal. + + :param other: The other object to check against. + + The value of :attr:`prereleases` is ignored. + + >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0,!=1.0.1") + True + >>> (SpecifierSet(">=1.0.0,!=1.0.1", prereleases=False) == + ... SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True)) + True + >>> SpecifierSet(">=1.0.0,!=1.0.1") == ">=1.0.0,!=1.0.1" + True + >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0") + False + >>> SpecifierSet(">=1.0.0,!=1.0.1") == SpecifierSet(">=1.0.0,!=1.0.2") + False + """ + if isinstance(other, (str, Specifier)): + other = SpecifierSet(str(other)) + elif not isinstance(other, SpecifierSet): + return NotImplemented + + return self._specs == other._specs + + def __len__(self) -> int: + """Returns the number of specifiers in this specifier set.""" + return len(self._specs) + + def __iter__(self) -> Iterator[Specifier]: + """ + Returns an iterator over all the underlying :class:`Specifier` instances + in this specifier set. + + >>> sorted(SpecifierSet(">=1.0.0,!=1.0.1"), key=str) + [, =1.0.0')>] + """ + return iter(self._specs) + + def __contains__(self, item: UnparsedVersion) -> bool: + """Return whether or not the item is contained in this specifier. + + :param item: The item to check for. + + This is used for the ``in`` operator and behaves the same as + :meth:`contains` with no ``prereleases`` argument passed. + + >>> "1.2.3" in SpecifierSet(">=1.0.0,!=1.0.1") + True + >>> Version("1.2.3") in SpecifierSet(">=1.0.0,!=1.0.1") + True + >>> "1.0.1" in SpecifierSet(">=1.0.0,!=1.0.1") + False + >>> "1.3.0a1" in SpecifierSet(">=1.0.0,!=1.0.1") + False + >>> "1.3.0a1" in SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True) + True + """ + return self.contains(item) + + def contains( + self, + item: UnparsedVersion, + prereleases: bool | None = None, + installed: bool | None = None, + ) -> bool: + """Return whether or not the item is contained in this SpecifierSet. + + :param item: + The item to check for, which can be a version string or a + :class:`Version` instance. + :param prereleases: + Whether or not to match prereleases with this SpecifierSet. If set to + ``None`` (the default), it uses :attr:`prereleases` to determine + whether or not prereleases are allowed. + + >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.2.3") + True + >>> SpecifierSet(">=1.0.0,!=1.0.1").contains(Version("1.2.3")) + True + >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.0.1") + False + >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.3.0a1") + False + >>> SpecifierSet(">=1.0.0,!=1.0.1", prereleases=True).contains("1.3.0a1") + True + >>> SpecifierSet(">=1.0.0,!=1.0.1").contains("1.3.0a1", prereleases=True) + True + """ + # Ensure that our item is a Version instance. + if not isinstance(item, Version): + item = Version(item) + + # Determine if we're forcing a prerelease or not, if we're not forcing + # one for this particular filter call, then we'll use whatever the + # SpecifierSet thinks for whether or not we should support prereleases. + if prereleases is None: + prereleases = self.prereleases + + # We can determine if we're going to allow pre-releases by looking to + # see if any of the underlying items supports them. If none of them do + # and this item is a pre-release then we do not allow it and we can + # short circuit that here. + # Note: This means that 1.0.dev1 would not be contained in something + # like >=1.0.devabc however it would be in >=1.0.debabc,>0.0.dev0 + if not prereleases and item.is_prerelease: + return False + + if installed and item.is_prerelease: + item = Version(item.base_version) + + # We simply dispatch to the underlying specs here to make sure that the + # given version is contained within all of them. + # Note: This use of all() here means that an empty set of specifiers + # will always return True, this is an explicit design decision. + return all(s.contains(item, prereleases=prereleases) for s in self._specs) + + def filter( + self, iterable: Iterable[UnparsedVersionVar], prereleases: bool | None = None + ) -> Iterator[UnparsedVersionVar]: + """Filter items in the given iterable, that match the specifiers in this set. + + :param iterable: + An iterable that can contain version strings and :class:`Version` instances. + The items in the iterable will be filtered according to the specifier. + :param prereleases: + Whether or not to allow prereleases in the returned iterator. If set to + ``None`` (the default), it will be intelligently decide whether to allow + prereleases or not (based on the :attr:`prereleases` attribute, and + whether the only versions matching are prereleases). + + This method is smarter than just ``filter(SpecifierSet(...).contains, [...])`` + because it implements the rule from :pep:`440` that a prerelease item + SHOULD be accepted if no other versions match the given specifier. + + >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.3", "1.5a1"])) + ['1.3'] + >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.3", Version("1.4")])) + ['1.3', ] + >>> list(SpecifierSet(">=1.2.3").filter(["1.2", "1.5a1"])) + [] + >>> list(SpecifierSet(">=1.2.3").filter(["1.3", "1.5a1"], prereleases=True)) + ['1.3', '1.5a1'] + >>> list(SpecifierSet(">=1.2.3", prereleases=True).filter(["1.3", "1.5a1"])) + ['1.3', '1.5a1'] + + An "empty" SpecifierSet will filter items based on the presence of prerelease + versions in the set. + + >>> list(SpecifierSet("").filter(["1.3", "1.5a1"])) + ['1.3'] + >>> list(SpecifierSet("").filter(["1.5a1"])) + ['1.5a1'] + >>> list(SpecifierSet("", prereleases=True).filter(["1.3", "1.5a1"])) + ['1.3', '1.5a1'] + >>> list(SpecifierSet("").filter(["1.3", "1.5a1"], prereleases=True)) + ['1.3', '1.5a1'] + """ + # Determine if we're forcing a prerelease or not, if we're not forcing + # one for this particular filter call, then we'll use whatever the + # SpecifierSet thinks for whether or not we should support prereleases. + if prereleases is None: + prereleases = self.prereleases + + # If we have any specifiers, then we want to wrap our iterable in the + # filter method for each one, this will act as a logical AND amongst + # each specifier. + if self._specs: + for spec in self._specs: + iterable = spec.filter(iterable, prereleases=bool(prereleases)) + return iter(iterable) + # If we do not have any specifiers, then we need to have a rough filter + # which will filter out any pre-releases, unless there are no final + # releases. + else: + filtered: list[UnparsedVersionVar] = [] + found_prereleases: list[UnparsedVersionVar] = [] + + for item in iterable: + parsed_version = _coerce_version(item) + + # Store any item which is a pre-release for later unless we've + # already found a final version or we are accepting prereleases + if parsed_version.is_prerelease and not prereleases: + if not filtered: + found_prereleases.append(item) + else: + filtered.append(item) + + # If we've found no items except for pre-releases, then we'll go + # ahead and use the pre-releases + if not filtered and found_prereleases and prereleases is None: + return iter(found_prereleases) + + return iter(filtered) diff --git a/lib/python3.1/site-packages/packaging/tags.py b/lib/python3.1/site-packages/packaging/tags.py new file mode 100644 index 0000000000000000000000000000000000000000..8522f59c4f2d0039c1a02ea9aef66fe017dbdb20 --- /dev/null +++ b/lib/python3.1/site-packages/packaging/tags.py @@ -0,0 +1,656 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import logging +import platform +import re +import struct +import subprocess +import sys +import sysconfig +from importlib.machinery import EXTENSION_SUFFIXES +from typing import ( + Iterable, + Iterator, + Sequence, + Tuple, + cast, +) + +from . import _manylinux, _musllinux + +logger = logging.getLogger(__name__) + +PythonVersion = Sequence[int] +AppleVersion = Tuple[int, int] + +INTERPRETER_SHORT_NAMES: dict[str, str] = { + "python": "py", # Generic. + "cpython": "cp", + "pypy": "pp", + "ironpython": "ip", + "jython": "jy", +} + + +_32_BIT_INTERPRETER = struct.calcsize("P") == 4 + + +class Tag: + """ + A representation of the tag triple for a wheel. + + Instances are considered immutable and thus are hashable. Equality checking + is also supported. + """ + + __slots__ = ["_abi", "_hash", "_interpreter", "_platform"] + + def __init__(self, interpreter: str, abi: str, platform: str) -> None: + self._interpreter = interpreter.lower() + self._abi = abi.lower() + self._platform = platform.lower() + # The __hash__ of every single element in a Set[Tag] will be evaluated each time + # that a set calls its `.disjoint()` method, which may be called hundreds of + # times when scanning a page of links for packages with tags matching that + # Set[Tag]. Pre-computing the value here produces significant speedups for + # downstream consumers. + self._hash = hash((self._interpreter, self._abi, self._platform)) + + @property + def interpreter(self) -> str: + return self._interpreter + + @property + def abi(self) -> str: + return self._abi + + @property + def platform(self) -> str: + return self._platform + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Tag): + return NotImplemented + + return ( + (self._hash == other._hash) # Short-circuit ASAP for perf reasons. + and (self._platform == other._platform) + and (self._abi == other._abi) + and (self._interpreter == other._interpreter) + ) + + def __hash__(self) -> int: + return self._hash + + def __str__(self) -> str: + return f"{self._interpreter}-{self._abi}-{self._platform}" + + def __repr__(self) -> str: + return f"<{self} @ {id(self)}>" + + +def parse_tag(tag: str) -> frozenset[Tag]: + """ + Parses the provided tag (e.g. `py3-none-any`) into a frozenset of Tag instances. + + Returning a set is required due to the possibility that the tag is a + compressed tag set. + """ + tags = set() + interpreters, abis, platforms = tag.split("-") + for interpreter in interpreters.split("."): + for abi in abis.split("."): + for platform_ in platforms.split("."): + tags.add(Tag(interpreter, abi, platform_)) + return frozenset(tags) + + +def _get_config_var(name: str, warn: bool = False) -> int | str | None: + value: int | str | None = sysconfig.get_config_var(name) + if value is None and warn: + logger.debug( + "Config variable '%s' is unset, Python ABI tag may be incorrect", name + ) + return value + + +def _normalize_string(string: str) -> str: + return string.replace(".", "_").replace("-", "_").replace(" ", "_") + + +def _is_threaded_cpython(abis: list[str]) -> bool: + """ + Determine if the ABI corresponds to a threaded (`--disable-gil`) build. + + The threaded builds are indicated by a "t" in the abiflags. + """ + if len(abis) == 0: + return False + # expect e.g., cp313 + m = re.match(r"cp\d+(.*)", abis[0]) + if not m: + return False + abiflags = m.group(1) + return "t" in abiflags + + +def _abi3_applies(python_version: PythonVersion, threading: bool) -> bool: + """ + Determine if the Python version supports abi3. + + PEP 384 was first implemented in Python 3.2. The threaded (`--disable-gil`) + builds do not support abi3. + """ + return len(python_version) > 1 and tuple(python_version) >= (3, 2) and not threading + + +def _cpython_abis(py_version: PythonVersion, warn: bool = False) -> list[str]: + py_version = tuple(py_version) # To allow for version comparison. + abis = [] + version = _version_nodot(py_version[:2]) + threading = debug = pymalloc = ucs4 = "" + with_debug = _get_config_var("Py_DEBUG", warn) + has_refcount = hasattr(sys, "gettotalrefcount") + # Windows doesn't set Py_DEBUG, so checking for support of debug-compiled + # extension modules is the best option. + # https://github.com/pypa/pip/issues/3383#issuecomment-173267692 + has_ext = "_d.pyd" in EXTENSION_SUFFIXES + if with_debug or (with_debug is None and (has_refcount or has_ext)): + debug = "d" + if py_version >= (3, 13) and _get_config_var("Py_GIL_DISABLED", warn): + threading = "t" + if py_version < (3, 8): + with_pymalloc = _get_config_var("WITH_PYMALLOC", warn) + if with_pymalloc or with_pymalloc is None: + pymalloc = "m" + if py_version < (3, 3): + unicode_size = _get_config_var("Py_UNICODE_SIZE", warn) + if unicode_size == 4 or ( + unicode_size is None and sys.maxunicode == 0x10FFFF + ): + ucs4 = "u" + elif debug: + # Debug builds can also load "normal" extension modules. + # We can also assume no UCS-4 or pymalloc requirement. + abis.append(f"cp{version}{threading}") + abis.insert(0, f"cp{version}{threading}{debug}{pymalloc}{ucs4}") + return abis + + +def cpython_tags( + python_version: PythonVersion | None = None, + abis: Iterable[str] | None = None, + platforms: Iterable[str] | None = None, + *, + warn: bool = False, +) -> Iterator[Tag]: + """ + Yields the tags for a CPython interpreter. + + The tags consist of: + - cp-- + - cp-abi3- + - cp-none- + - cp-abi3- # Older Python versions down to 3.2. + + If python_version only specifies a major version then user-provided ABIs and + the 'none' ABItag will be used. + + If 'abi3' or 'none' are specified in 'abis' then they will be yielded at + their normal position and not at the beginning. + """ + if not python_version: + python_version = sys.version_info[:2] + + interpreter = f"cp{_version_nodot(python_version[:2])}" + + if abis is None: + if len(python_version) > 1: + abis = _cpython_abis(python_version, warn) + else: + abis = [] + abis = list(abis) + # 'abi3' and 'none' are explicitly handled later. + for explicit_abi in ("abi3", "none"): + try: + abis.remove(explicit_abi) + except ValueError: + pass + + platforms = list(platforms or platform_tags()) + for abi in abis: + for platform_ in platforms: + yield Tag(interpreter, abi, platform_) + + threading = _is_threaded_cpython(abis) + use_abi3 = _abi3_applies(python_version, threading) + if use_abi3: + yield from (Tag(interpreter, "abi3", platform_) for platform_ in platforms) + yield from (Tag(interpreter, "none", platform_) for platform_ in platforms) + + if use_abi3: + for minor_version in range(python_version[1] - 1, 1, -1): + for platform_ in platforms: + version = _version_nodot((python_version[0], minor_version)) + interpreter = f"cp{version}" + yield Tag(interpreter, "abi3", platform_) + + +def _generic_abi() -> list[str]: + """ + Return the ABI tag based on EXT_SUFFIX. + """ + # The following are examples of `EXT_SUFFIX`. + # We want to keep the parts which are related to the ABI and remove the + # parts which are related to the platform: + # - linux: '.cpython-310-x86_64-linux-gnu.so' => cp310 + # - mac: '.cpython-310-darwin.so' => cp310 + # - win: '.cp310-win_amd64.pyd' => cp310 + # - win: '.pyd' => cp37 (uses _cpython_abis()) + # - pypy: '.pypy38-pp73-x86_64-linux-gnu.so' => pypy38_pp73 + # - graalpy: '.graalpy-38-native-x86_64-darwin.dylib' + # => graalpy_38_native + + ext_suffix = _get_config_var("EXT_SUFFIX", warn=True) + if not isinstance(ext_suffix, str) or ext_suffix[0] != ".": + raise SystemError("invalid sysconfig.get_config_var('EXT_SUFFIX')") + parts = ext_suffix.split(".") + if len(parts) < 3: + # CPython3.7 and earlier uses ".pyd" on Windows. + return _cpython_abis(sys.version_info[:2]) + soabi = parts[1] + if soabi.startswith("cpython"): + # non-windows + abi = "cp" + soabi.split("-")[1] + elif soabi.startswith("cp"): + # windows + abi = soabi.split("-")[0] + elif soabi.startswith("pypy"): + abi = "-".join(soabi.split("-")[:2]) + elif soabi.startswith("graalpy"): + abi = "-".join(soabi.split("-")[:3]) + elif soabi: + # pyston, ironpython, others? + abi = soabi + else: + return [] + return [_normalize_string(abi)] + + +def generic_tags( + interpreter: str | None = None, + abis: Iterable[str] | None = None, + platforms: Iterable[str] | None = None, + *, + warn: bool = False, +) -> Iterator[Tag]: + """ + Yields the tags for a generic interpreter. + + The tags consist of: + - -- + + The "none" ABI will be added if it was not explicitly provided. + """ + if not interpreter: + interp_name = interpreter_name() + interp_version = interpreter_version(warn=warn) + interpreter = "".join([interp_name, interp_version]) + if abis is None: + abis = _generic_abi() + else: + abis = list(abis) + platforms = list(platforms or platform_tags()) + if "none" not in abis: + abis.append("none") + for abi in abis: + for platform_ in platforms: + yield Tag(interpreter, abi, platform_) + + +def _py_interpreter_range(py_version: PythonVersion) -> Iterator[str]: + """ + Yields Python versions in descending order. + + After the latest version, the major-only version will be yielded, and then + all previous versions of that major version. + """ + if len(py_version) > 1: + yield f"py{_version_nodot(py_version[:2])}" + yield f"py{py_version[0]}" + if len(py_version) > 1: + for minor in range(py_version[1] - 1, -1, -1): + yield f"py{_version_nodot((py_version[0], minor))}" + + +def compatible_tags( + python_version: PythonVersion | None = None, + interpreter: str | None = None, + platforms: Iterable[str] | None = None, +) -> Iterator[Tag]: + """ + Yields the sequence of tags that are compatible with a specific version of Python. + + The tags consist of: + - py*-none- + - -none-any # ... if `interpreter` is provided. + - py*-none-any + """ + if not python_version: + python_version = sys.version_info[:2] + platforms = list(platforms or platform_tags()) + for version in _py_interpreter_range(python_version): + for platform_ in platforms: + yield Tag(version, "none", platform_) + if interpreter: + yield Tag(interpreter, "none", "any") + for version in _py_interpreter_range(python_version): + yield Tag(version, "none", "any") + + +def _mac_arch(arch: str, is_32bit: bool = _32_BIT_INTERPRETER) -> str: + if not is_32bit: + return arch + + if arch.startswith("ppc"): + return "ppc" + + return "i386" + + +def _mac_binary_formats(version: AppleVersion, cpu_arch: str) -> list[str]: + formats = [cpu_arch] + if cpu_arch == "x86_64": + if version < (10, 4): + return [] + formats.extend(["intel", "fat64", "fat32"]) + + elif cpu_arch == "i386": + if version < (10, 4): + return [] + formats.extend(["intel", "fat32", "fat"]) + + elif cpu_arch == "ppc64": + # TODO: Need to care about 32-bit PPC for ppc64 through 10.2? + if version > (10, 5) or version < (10, 4): + return [] + formats.append("fat64") + + elif cpu_arch == "ppc": + if version > (10, 6): + return [] + formats.extend(["fat32", "fat"]) + + if cpu_arch in {"arm64", "x86_64"}: + formats.append("universal2") + + if cpu_arch in {"x86_64", "i386", "ppc64", "ppc", "intel"}: + formats.append("universal") + + return formats + + +def mac_platforms( + version: AppleVersion | None = None, arch: str | None = None +) -> Iterator[str]: + """ + Yields the platform tags for a macOS system. + + The `version` parameter is a two-item tuple specifying the macOS version to + generate platform tags for. The `arch` parameter is the CPU architecture to + generate platform tags for. Both parameters default to the appropriate value + for the current system. + """ + version_str, _, cpu_arch = platform.mac_ver() + if version is None: + version = cast("AppleVersion", tuple(map(int, version_str.split(".")[:2]))) + if version == (10, 16): + # When built against an older macOS SDK, Python will report macOS 10.16 + # instead of the real version. + version_str = subprocess.run( + [ + sys.executable, + "-sS", + "-c", + "import platform; print(platform.mac_ver()[0])", + ], + check=True, + env={"SYSTEM_VERSION_COMPAT": "0"}, + stdout=subprocess.PIPE, + text=True, + ).stdout + version = cast("AppleVersion", tuple(map(int, version_str.split(".")[:2]))) + else: + version = version + if arch is None: + arch = _mac_arch(cpu_arch) + else: + arch = arch + + if (10, 0) <= version and version < (11, 0): + # Prior to Mac OS 11, each yearly release of Mac OS bumped the + # "minor" version number. The major version was always 10. + major_version = 10 + for minor_version in range(version[1], -1, -1): + compat_version = major_version, minor_version + binary_formats = _mac_binary_formats(compat_version, arch) + for binary_format in binary_formats: + yield f"macosx_{major_version}_{minor_version}_{binary_format}" + + if version >= (11, 0): + # Starting with Mac OS 11, each yearly release bumps the major version + # number. The minor versions are now the midyear updates. + minor_version = 0 + for major_version in range(version[0], 10, -1): + compat_version = major_version, minor_version + binary_formats = _mac_binary_formats(compat_version, arch) + for binary_format in binary_formats: + yield f"macosx_{major_version}_{minor_version}_{binary_format}" + + if version >= (11, 0): + # Mac OS 11 on x86_64 is compatible with binaries from previous releases. + # Arm64 support was introduced in 11.0, so no Arm binaries from previous + # releases exist. + # + # However, the "universal2" binary format can have a + # macOS version earlier than 11.0 when the x86_64 part of the binary supports + # that version of macOS. + major_version = 10 + if arch == "x86_64": + for minor_version in range(16, 3, -1): + compat_version = major_version, minor_version + binary_formats = _mac_binary_formats(compat_version, arch) + for binary_format in binary_formats: + yield f"macosx_{major_version}_{minor_version}_{binary_format}" + else: + for minor_version in range(16, 3, -1): + compat_version = major_version, minor_version + binary_format = "universal2" + yield f"macosx_{major_version}_{minor_version}_{binary_format}" + + +def ios_platforms( + version: AppleVersion | None = None, multiarch: str | None = None +) -> Iterator[str]: + """ + Yields the platform tags for an iOS system. + + :param version: A two-item tuple specifying the iOS version to generate + platform tags for. Defaults to the current iOS version. + :param multiarch: The CPU architecture+ABI to generate platform tags for - + (the value used by `sys.implementation._multiarch` e.g., + `arm64_iphoneos` or `x84_64_iphonesimulator`). Defaults to the current + multiarch value. + """ + if version is None: + # if iOS is the current platform, ios_ver *must* be defined. However, + # it won't exist for CPython versions before 3.13, which causes a mypy + # error. + _, release, _, _ = platform.ios_ver() # type: ignore[attr-defined, unused-ignore] + version = cast("AppleVersion", tuple(map(int, release.split(".")[:2]))) + + if multiarch is None: + multiarch = sys.implementation._multiarch + multiarch = multiarch.replace("-", "_") + + ios_platform_template = "ios_{major}_{minor}_{multiarch}" + + # Consider any iOS major.minor version from the version requested, down to + # 12.0. 12.0 is the first iOS version that is known to have enough features + # to support CPython. Consider every possible minor release up to X.9. There + # highest the minor has ever gone is 8 (14.8 and 15.8) but having some extra + # candidates that won't ever match doesn't really hurt, and it saves us from + # having to keep an explicit list of known iOS versions in the code. Return + # the results descending order of version number. + + # If the requested major version is less than 12, there won't be any matches. + if version[0] < 12: + return + + # Consider the actual X.Y version that was requested. + yield ios_platform_template.format( + major=version[0], minor=version[1], multiarch=multiarch + ) + + # Consider every minor version from X.0 to the minor version prior to the + # version requested by the platform. + for minor in range(version[1] - 1, -1, -1): + yield ios_platform_template.format( + major=version[0], minor=minor, multiarch=multiarch + ) + + for major in range(version[0] - 1, 11, -1): + for minor in range(9, -1, -1): + yield ios_platform_template.format( + major=major, minor=minor, multiarch=multiarch + ) + + +def android_platforms( + api_level: int | None = None, abi: str | None = None +) -> Iterator[str]: + """ + Yields the :attr:`~Tag.platform` tags for Android. If this function is invoked on + non-Android platforms, the ``api_level`` and ``abi`` arguments are required. + + :param int api_level: The maximum `API level + `__ to return. Defaults + to the current system's version, as returned by ``platform.android_ver``. + :param str abi: The `Android ABI `__, + e.g. ``arm64_v8a``. Defaults to the current system's ABI , as returned by + ``sysconfig.get_platform``. Hyphens and periods will be replaced with + underscores. + """ + if platform.system() != "Android" and (api_level is None or abi is None): + raise TypeError( + "on non-Android platforms, the api_level and abi arguments are required" + ) + + if api_level is None: + # Python 3.13 was the first version to return platform.system() == "Android", + # and also the first version to define platform.android_ver(). + api_level = platform.android_ver().api_level # type: ignore[attr-defined] + + if abi is None: + abi = sysconfig.get_platform().split("-")[-1] + abi = _normalize_string(abi) + + # 16 is the minimum API level known to have enough features to support CPython + # without major patching. Yield every API level from the maximum down to the + # minimum, inclusive. + min_api_level = 16 + for ver in range(api_level, min_api_level - 1, -1): + yield f"android_{ver}_{abi}" + + +def _linux_platforms(is_32bit: bool = _32_BIT_INTERPRETER) -> Iterator[str]: + linux = _normalize_string(sysconfig.get_platform()) + if not linux.startswith("linux_"): + # we should never be here, just yield the sysconfig one and return + yield linux + return + if is_32bit: + if linux == "linux_x86_64": + linux = "linux_i686" + elif linux == "linux_aarch64": + linux = "linux_armv8l" + _, arch = linux.split("_", 1) + archs = {"armv8l": ["armv8l", "armv7l"]}.get(arch, [arch]) + yield from _manylinux.platform_tags(archs) + yield from _musllinux.platform_tags(archs) + for arch in archs: + yield f"linux_{arch}" + + +def _generic_platforms() -> Iterator[str]: + yield _normalize_string(sysconfig.get_platform()) + + +def platform_tags() -> Iterator[str]: + """ + Provides the platform tags for this installation. + """ + if platform.system() == "Darwin": + return mac_platforms() + elif platform.system() == "iOS": + return ios_platforms() + elif platform.system() == "Android": + return android_platforms() + elif platform.system() == "Linux": + return _linux_platforms() + else: + return _generic_platforms() + + +def interpreter_name() -> str: + """ + Returns the name of the running interpreter. + + Some implementations have a reserved, two-letter abbreviation which will + be returned when appropriate. + """ + name = sys.implementation.name + return INTERPRETER_SHORT_NAMES.get(name) or name + + +def interpreter_version(*, warn: bool = False) -> str: + """ + Returns the version of the running interpreter. + """ + version = _get_config_var("py_version_nodot", warn=warn) + if version: + version = str(version) + else: + version = _version_nodot(sys.version_info[:2]) + return version + + +def _version_nodot(version: PythonVersion) -> str: + return "".join(map(str, version)) + + +def sys_tags(*, warn: bool = False) -> Iterator[Tag]: + """ + Returns the sequence of tag triples for the running interpreter. + + The order of the sequence corresponds to priority order for the + interpreter, from most to least important. + """ + + interp_name = interpreter_name() + if interp_name == "cp": + yield from cpython_tags(warn=warn) + else: + yield from generic_tags() + + if interp_name == "pp": + interp = "pp3" + elif interp_name == "cp": + interp = "cp" + interpreter_version(warn=warn) + else: + interp = None + yield from compatible_tags(interpreter=interp) diff --git a/lib/python3.1/site-packages/packaging/utils.py b/lib/python3.1/site-packages/packaging/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..23450953df74eccd9c13cd2a955ce09d1f968565 --- /dev/null +++ b/lib/python3.1/site-packages/packaging/utils.py @@ -0,0 +1,163 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import functools +import re +from typing import NewType, Tuple, Union, cast + +from .tags import Tag, parse_tag +from .version import InvalidVersion, Version, _TrimmedRelease + +BuildTag = Union[Tuple[()], Tuple[int, str]] +NormalizedName = NewType("NormalizedName", str) + + +class InvalidName(ValueError): + """ + An invalid distribution name; users should refer to the packaging user guide. + """ + + +class InvalidWheelFilename(ValueError): + """ + An invalid wheel filename was found, users should refer to PEP 427. + """ + + +class InvalidSdistFilename(ValueError): + """ + An invalid sdist filename was found, users should refer to the packaging user guide. + """ + + +# Core metadata spec for `Name` +_validate_regex = re.compile( + r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", re.IGNORECASE +) +_canonicalize_regex = re.compile(r"[-_.]+") +_normalized_regex = re.compile(r"^([a-z0-9]|[a-z0-9]([a-z0-9-](?!--))*[a-z0-9])$") +# PEP 427: The build number must start with a digit. +_build_tag_regex = re.compile(r"(\d+)(.*)") + + +def canonicalize_name(name: str, *, validate: bool = False) -> NormalizedName: + if validate and not _validate_regex.match(name): + raise InvalidName(f"name is invalid: {name!r}") + # This is taken from PEP 503. + value = _canonicalize_regex.sub("-", name).lower() + return cast(NormalizedName, value) + + +def is_normalized_name(name: str) -> bool: + return _normalized_regex.match(name) is not None + + +@functools.singledispatch +def canonicalize_version( + version: Version | str, *, strip_trailing_zero: bool = True +) -> str: + """ + Return a canonical form of a version as a string. + + >>> canonicalize_version('1.0.1') + '1.0.1' + + Per PEP 625, versions may have multiple canonical forms, differing + only by trailing zeros. + + >>> canonicalize_version('1.0.0') + '1' + >>> canonicalize_version('1.0.0', strip_trailing_zero=False) + '1.0.0' + + Invalid versions are returned unaltered. + + >>> canonicalize_version('foo bar baz') + 'foo bar baz' + """ + return str(_TrimmedRelease(str(version)) if strip_trailing_zero else version) + + +@canonicalize_version.register +def _(version: str, *, strip_trailing_zero: bool = True) -> str: + try: + parsed = Version(version) + except InvalidVersion: + # Legacy versions cannot be normalized + return version + return canonicalize_version(parsed, strip_trailing_zero=strip_trailing_zero) + + +def parse_wheel_filename( + filename: str, +) -> tuple[NormalizedName, Version, BuildTag, frozenset[Tag]]: + if not filename.endswith(".whl"): + raise InvalidWheelFilename( + f"Invalid wheel filename (extension must be '.whl'): {filename!r}" + ) + + filename = filename[:-4] + dashes = filename.count("-") + if dashes not in (4, 5): + raise InvalidWheelFilename( + f"Invalid wheel filename (wrong number of parts): {filename!r}" + ) + + parts = filename.split("-", dashes - 2) + name_part = parts[0] + # See PEP 427 for the rules on escaping the project name. + if "__" in name_part or re.match(r"^[\w\d._]*$", name_part, re.UNICODE) is None: + raise InvalidWheelFilename(f"Invalid project name: {filename!r}") + name = canonicalize_name(name_part) + + try: + version = Version(parts[1]) + except InvalidVersion as e: + raise InvalidWheelFilename( + f"Invalid wheel filename (invalid version): {filename!r}" + ) from e + + if dashes == 5: + build_part = parts[2] + build_match = _build_tag_regex.match(build_part) + if build_match is None: + raise InvalidWheelFilename( + f"Invalid build number: {build_part} in {filename!r}" + ) + build = cast(BuildTag, (int(build_match.group(1)), build_match.group(2))) + else: + build = () + tags = parse_tag(parts[-1]) + return (name, version, build, tags) + + +def parse_sdist_filename(filename: str) -> tuple[NormalizedName, Version]: + if filename.endswith(".tar.gz"): + file_stem = filename[: -len(".tar.gz")] + elif filename.endswith(".zip"): + file_stem = filename[: -len(".zip")] + else: + raise InvalidSdistFilename( + f"Invalid sdist filename (extension must be '.tar.gz' or '.zip'):" + f" {filename!r}" + ) + + # We are requiring a PEP 440 version, which cannot contain dashes, + # so we split on the last dash. + name_part, sep, version_part = file_stem.rpartition("-") + if not sep: + raise InvalidSdistFilename(f"Invalid sdist filename: {filename!r}") + + name = canonicalize_name(name_part) + + try: + version = Version(version_part) + except InvalidVersion as e: + raise InvalidSdistFilename( + f"Invalid sdist filename (invalid version): {filename!r}" + ) from e + + return (name, version) diff --git a/lib/python3.1/site-packages/packaging/version.py b/lib/python3.1/site-packages/packaging/version.py new file mode 100644 index 0000000000000000000000000000000000000000..c9bbda20e463b8d9389ecd65f74af33810a02bdd --- /dev/null +++ b/lib/python3.1/site-packages/packaging/version.py @@ -0,0 +1,582 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +""" +.. testsetup:: + + from packaging.version import parse, Version +""" + +from __future__ import annotations + +import itertools +import re +from typing import Any, Callable, NamedTuple, SupportsInt, Tuple, Union + +from ._structures import Infinity, InfinityType, NegativeInfinity, NegativeInfinityType + +__all__ = ["VERSION_PATTERN", "InvalidVersion", "Version", "parse"] + +LocalType = Tuple[Union[int, str], ...] + +CmpPrePostDevType = Union[InfinityType, NegativeInfinityType, Tuple[str, int]] +CmpLocalType = Union[ + NegativeInfinityType, + Tuple[Union[Tuple[int, str], Tuple[NegativeInfinityType, Union[int, str]]], ...], +] +CmpKey = Tuple[ + int, + Tuple[int, ...], + CmpPrePostDevType, + CmpPrePostDevType, + CmpPrePostDevType, + CmpLocalType, +] +VersionComparisonMethod = Callable[[CmpKey, CmpKey], bool] + + +class _Version(NamedTuple): + epoch: int + release: tuple[int, ...] + dev: tuple[str, int] | None + pre: tuple[str, int] | None + post: tuple[str, int] | None + local: LocalType | None + + +def parse(version: str) -> Version: + """Parse the given version string. + + >>> parse('1.0.dev1') + + + :param version: The version string to parse. + :raises InvalidVersion: When the version string is not a valid version. + """ + return Version(version) + + +class InvalidVersion(ValueError): + """Raised when a version string is not a valid version. + + >>> Version("invalid") + Traceback (most recent call last): + ... + packaging.version.InvalidVersion: Invalid version: 'invalid' + """ + + +class _BaseVersion: + _key: tuple[Any, ...] + + def __hash__(self) -> int: + return hash(self._key) + + # Please keep the duplicated `isinstance` check + # in the six comparisons hereunder + # unless you find a way to avoid adding overhead function calls. + def __lt__(self, other: _BaseVersion) -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key < other._key + + def __le__(self, other: _BaseVersion) -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key <= other._key + + def __eq__(self, other: object) -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key == other._key + + def __ge__(self, other: _BaseVersion) -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key >= other._key + + def __gt__(self, other: _BaseVersion) -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key > other._key + + def __ne__(self, other: object) -> bool: + if not isinstance(other, _BaseVersion): + return NotImplemented + + return self._key != other._key + + +# Deliberately not anchored to the start and end of the string, to make it +# easier for 3rd party code to reuse +_VERSION_PATTERN = r""" + v? + (?: + (?:(?P[0-9]+)!)? # epoch + (?P[0-9]+(?:\.[0-9]+)*) # release segment + (?P
                                              # pre-release
+            [-_\.]?
+            (?Palpha|a|beta|b|preview|pre|c|rc)
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+        (?P                                         # post release
+            (?:-(?P[0-9]+))
+            |
+            (?:
+                [-_\.]?
+                (?Ppost|rev|r)
+                [-_\.]?
+                (?P[0-9]+)?
+            )
+        )?
+        (?P                                          # dev release
+            [-_\.]?
+            (?Pdev)
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+    )
+    (?:\+(?P[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
+"""
+
+VERSION_PATTERN = _VERSION_PATTERN
+"""
+A string containing the regular expression used to match a valid version.
+
+The pattern is not anchored at either end, and is intended for embedding in larger
+expressions (for example, matching a version number as part of a file name). The
+regular expression should be compiled with the ``re.VERBOSE`` and ``re.IGNORECASE``
+flags set.
+
+:meta hide-value:
+"""
+
+
+class Version(_BaseVersion):
+    """This class abstracts handling of a project's versions.
+
+    A :class:`Version` instance is comparison aware and can be compared and
+    sorted using the standard Python interfaces.
+
+    >>> v1 = Version("1.0a5")
+    >>> v2 = Version("1.0")
+    >>> v1
+    
+    >>> v2
+    
+    >>> v1 < v2
+    True
+    >>> v1 == v2
+    False
+    >>> v1 > v2
+    False
+    >>> v1 >= v2
+    False
+    >>> v1 <= v2
+    True
+    """
+
+    _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE)
+    _key: CmpKey
+
+    def __init__(self, version: str) -> None:
+        """Initialize a Version object.
+
+        :param version:
+            The string representation of a version which will be parsed and normalized
+            before use.
+        :raises InvalidVersion:
+            If the ``version`` does not conform to PEP 440 in any way then this
+            exception will be raised.
+        """
+
+        # Validate the version and parse it into pieces
+        match = self._regex.search(version)
+        if not match:
+            raise InvalidVersion(f"Invalid version: {version!r}")
+
+        # Store the parsed out pieces of the version
+        self._version = _Version(
+            epoch=int(match.group("epoch")) if match.group("epoch") else 0,
+            release=tuple(int(i) for i in match.group("release").split(".")),
+            pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")),
+            post=_parse_letter_version(
+                match.group("post_l"), match.group("post_n1") or match.group("post_n2")
+            ),
+            dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")),
+            local=_parse_local_version(match.group("local")),
+        )
+
+        # Generate a key which will be used for sorting
+        self._key = _cmpkey(
+            self._version.epoch,
+            self._version.release,
+            self._version.pre,
+            self._version.post,
+            self._version.dev,
+            self._version.local,
+        )
+
+    def __repr__(self) -> str:
+        """A representation of the Version that shows all internal state.
+
+        >>> Version('1.0.0')
+        
+        """
+        return f""
+
+    def __str__(self) -> str:
+        """A string representation of the version that can be round-tripped.
+
+        >>> str(Version("1.0a5"))
+        '1.0a5'
+        """
+        parts = []
+
+        # Epoch
+        if self.epoch != 0:
+            parts.append(f"{self.epoch}!")
+
+        # Release segment
+        parts.append(".".join(str(x) for x in self.release))
+
+        # Pre-release
+        if self.pre is not None:
+            parts.append("".join(str(x) for x in self.pre))
+
+        # Post-release
+        if self.post is not None:
+            parts.append(f".post{self.post}")
+
+        # Development release
+        if self.dev is not None:
+            parts.append(f".dev{self.dev}")
+
+        # Local version segment
+        if self.local is not None:
+            parts.append(f"+{self.local}")
+
+        return "".join(parts)
+
+    @property
+    def epoch(self) -> int:
+        """The epoch of the version.
+
+        >>> Version("2.0.0").epoch
+        0
+        >>> Version("1!2.0.0").epoch
+        1
+        """
+        return self._version.epoch
+
+    @property
+    def release(self) -> tuple[int, ...]:
+        """The components of the "release" segment of the version.
+
+        >>> Version("1.2.3").release
+        (1, 2, 3)
+        >>> Version("2.0.0").release
+        (2, 0, 0)
+        >>> Version("1!2.0.0.post0").release
+        (2, 0, 0)
+
+        Includes trailing zeroes but not the epoch or any pre-release / development /
+        post-release suffixes.
+        """
+        return self._version.release
+
+    @property
+    def pre(self) -> tuple[str, int] | None:
+        """The pre-release segment of the version.
+
+        >>> print(Version("1.2.3").pre)
+        None
+        >>> Version("1.2.3a1").pre
+        ('a', 1)
+        >>> Version("1.2.3b1").pre
+        ('b', 1)
+        >>> Version("1.2.3rc1").pre
+        ('rc', 1)
+        """
+        return self._version.pre
+
+    @property
+    def post(self) -> int | None:
+        """The post-release number of the version.
+
+        >>> print(Version("1.2.3").post)
+        None
+        >>> Version("1.2.3.post1").post
+        1
+        """
+        return self._version.post[1] if self._version.post else None
+
+    @property
+    def dev(self) -> int | None:
+        """The development number of the version.
+
+        >>> print(Version("1.2.3").dev)
+        None
+        >>> Version("1.2.3.dev1").dev
+        1
+        """
+        return self._version.dev[1] if self._version.dev else None
+
+    @property
+    def local(self) -> str | None:
+        """The local version segment of the version.
+
+        >>> print(Version("1.2.3").local)
+        None
+        >>> Version("1.2.3+abc").local
+        'abc'
+        """
+        if self._version.local:
+            return ".".join(str(x) for x in self._version.local)
+        else:
+            return None
+
+    @property
+    def public(self) -> str:
+        """The public portion of the version.
+
+        >>> Version("1.2.3").public
+        '1.2.3'
+        >>> Version("1.2.3+abc").public
+        '1.2.3'
+        >>> Version("1!1.2.3dev1+abc").public
+        '1!1.2.3.dev1'
+        """
+        return str(self).split("+", 1)[0]
+
+    @property
+    def base_version(self) -> str:
+        """The "base version" of the version.
+
+        >>> Version("1.2.3").base_version
+        '1.2.3'
+        >>> Version("1.2.3+abc").base_version
+        '1.2.3'
+        >>> Version("1!1.2.3dev1+abc").base_version
+        '1!1.2.3'
+
+        The "base version" is the public version of the project without any pre or post
+        release markers.
+        """
+        parts = []
+
+        # Epoch
+        if self.epoch != 0:
+            parts.append(f"{self.epoch}!")
+
+        # Release segment
+        parts.append(".".join(str(x) for x in self.release))
+
+        return "".join(parts)
+
+    @property
+    def is_prerelease(self) -> bool:
+        """Whether this version is a pre-release.
+
+        >>> Version("1.2.3").is_prerelease
+        False
+        >>> Version("1.2.3a1").is_prerelease
+        True
+        >>> Version("1.2.3b1").is_prerelease
+        True
+        >>> Version("1.2.3rc1").is_prerelease
+        True
+        >>> Version("1.2.3dev1").is_prerelease
+        True
+        """
+        return self.dev is not None or self.pre is not None
+
+    @property
+    def is_postrelease(self) -> bool:
+        """Whether this version is a post-release.
+
+        >>> Version("1.2.3").is_postrelease
+        False
+        >>> Version("1.2.3.post1").is_postrelease
+        True
+        """
+        return self.post is not None
+
+    @property
+    def is_devrelease(self) -> bool:
+        """Whether this version is a development release.
+
+        >>> Version("1.2.3").is_devrelease
+        False
+        >>> Version("1.2.3.dev1").is_devrelease
+        True
+        """
+        return self.dev is not None
+
+    @property
+    def major(self) -> int:
+        """The first item of :attr:`release` or ``0`` if unavailable.
+
+        >>> Version("1.2.3").major
+        1
+        """
+        return self.release[0] if len(self.release) >= 1 else 0
+
+    @property
+    def minor(self) -> int:
+        """The second item of :attr:`release` or ``0`` if unavailable.
+
+        >>> Version("1.2.3").minor
+        2
+        >>> Version("1").minor
+        0
+        """
+        return self.release[1] if len(self.release) >= 2 else 0
+
+    @property
+    def micro(self) -> int:
+        """The third item of :attr:`release` or ``0`` if unavailable.
+
+        >>> Version("1.2.3").micro
+        3
+        >>> Version("1").micro
+        0
+        """
+        return self.release[2] if len(self.release) >= 3 else 0
+
+
+class _TrimmedRelease(Version):
+    @property
+    def release(self) -> tuple[int, ...]:
+        """
+        Release segment without any trailing zeros.
+
+        >>> _TrimmedRelease('1.0.0').release
+        (1,)
+        >>> _TrimmedRelease('0.0').release
+        (0,)
+        """
+        rel = super().release
+        nonzeros = (index for index, val in enumerate(rel) if val)
+        last_nonzero = max(nonzeros, default=0)
+        return rel[: last_nonzero + 1]
+
+
+def _parse_letter_version(
+    letter: str | None, number: str | bytes | SupportsInt | None
+) -> tuple[str, int] | None:
+    if letter:
+        # We consider there to be an implicit 0 in a pre-release if there is
+        # not a numeral associated with it.
+        if number is None:
+            number = 0
+
+        # We normalize any letters to their lower case form
+        letter = letter.lower()
+
+        # We consider some words to be alternate spellings of other words and
+        # in those cases we want to normalize the spellings to our preferred
+        # spelling.
+        if letter == "alpha":
+            letter = "a"
+        elif letter == "beta":
+            letter = "b"
+        elif letter in ["c", "pre", "preview"]:
+            letter = "rc"
+        elif letter in ["rev", "r"]:
+            letter = "post"
+
+        return letter, int(number)
+
+    assert not letter
+    if number:
+        # We assume if we are given a number, but we are not given a letter
+        # then this is using the implicit post release syntax (e.g. 1.0-1)
+        letter = "post"
+
+        return letter, int(number)
+
+    return None
+
+
+_local_version_separators = re.compile(r"[\._-]")
+
+
+def _parse_local_version(local: str | None) -> LocalType | None:
+    """
+    Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve").
+    """
+    if local is not None:
+        return tuple(
+            part.lower() if not part.isdigit() else int(part)
+            for part in _local_version_separators.split(local)
+        )
+    return None
+
+
+def _cmpkey(
+    epoch: int,
+    release: tuple[int, ...],
+    pre: tuple[str, int] | None,
+    post: tuple[str, int] | None,
+    dev: tuple[str, int] | None,
+    local: LocalType | None,
+) -> CmpKey:
+    # When we compare a release version, we want to compare it with all of the
+    # trailing zeros removed. So we'll use a reverse the list, drop all the now
+    # leading zeros until we come to something non zero, then take the rest
+    # re-reverse it back into the correct order and make it a tuple and use
+    # that for our sorting key.
+    _release = tuple(
+        reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release))))
+    )
+
+    # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0.
+    # We'll do this by abusing the pre segment, but we _only_ want to do this
+    # if there is not a pre or a post segment. If we have one of those then
+    # the normal sorting rules will handle this case correctly.
+    if pre is None and post is None and dev is not None:
+        _pre: CmpPrePostDevType = NegativeInfinity
+    # Versions without a pre-release (except as noted above) should sort after
+    # those with one.
+    elif pre is None:
+        _pre = Infinity
+    else:
+        _pre = pre
+
+    # Versions without a post segment should sort before those with one.
+    if post is None:
+        _post: CmpPrePostDevType = NegativeInfinity
+
+    else:
+        _post = post
+
+    # Versions without a development segment should sort after those with one.
+    if dev is None:
+        _dev: CmpPrePostDevType = Infinity
+
+    else:
+        _dev = dev
+
+    if local is None:
+        # Versions without a local segment should sort before those with one.
+        _local: CmpLocalType = NegativeInfinity
+    else:
+        # Versions with a local segment need that segment parsed to implement
+        # the sorting rules in PEP440.
+        # - Alpha numeric segments sort before numeric segments
+        # - Alpha numeric segments sort lexicographically
+        # - Numeric segments sort numerically
+        # - Shorter versions sort before longer versions when the prefixes
+        #   match exactly
+        _local = tuple(
+            (i, "") if isinstance(i, int) else (NegativeInfinity, i) for i in local
+        )
+
+    return epoch, _release, _pre, _post, _dev, _local
diff --git a/lib/python3.1/site-packages/pip-25.2.dist-info/INSTALLER b/lib/python3.1/site-packages/pip-25.2.dist-info/INSTALLER
new file mode 100644
index 0000000000000000000000000000000000000000..f79e4cb9aaf0b2d9e8ba78861e2071317b2384b3
--- /dev/null
+++ b/lib/python3.1/site-packages/pip-25.2.dist-info/INSTALLER
@@ -0,0 +1 @@
+conda
\ No newline at end of file
diff --git a/lib/python3.1/site-packages/pip-25.2.dist-info/METADATA b/lib/python3.1/site-packages/pip-25.2.dist-info/METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..2cf904bc205ba3bfbbd5ab65edeab062bddcb609
--- /dev/null
+++ b/lib/python3.1/site-packages/pip-25.2.dist-info/METADATA
@@ -0,0 +1,112 @@
+Metadata-Version: 2.4
+Name: pip
+Version: 25.2
+Summary: The PyPA recommended tool for installing Python packages.
+Author-email: The pip developers 
+License-Expression: MIT
+Project-URL: Homepage, https://pip.pypa.io/
+Project-URL: Documentation, https://pip.pypa.io
+Project-URL: Source, https://github.com/pypa/pip
+Project-URL: Changelog, https://pip.pypa.io/en/stable/news/
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=3.9
+Description-Content-Type: text/x-rst
+License-File: AUTHORS.txt
+License-File: LICENSE.txt
+License-File: src/pip/_vendor/msgpack/COPYING
+License-File: src/pip/_vendor/cachecontrol/LICENSE.txt
+License-File: src/pip/_vendor/certifi/LICENSE
+License-File: src/pip/_vendor/dependency_groups/LICENSE.txt
+License-File: src/pip/_vendor/distlib/LICENSE.txt
+License-File: src/pip/_vendor/distro/LICENSE
+License-File: src/pip/_vendor/idna/LICENSE.md
+License-File: src/pip/_vendor/packaging/LICENSE
+License-File: src/pip/_vendor/packaging/LICENSE.APACHE
+License-File: src/pip/_vendor/packaging/LICENSE.BSD
+License-File: src/pip/_vendor/pkg_resources/LICENSE
+License-File: src/pip/_vendor/platformdirs/LICENSE
+License-File: src/pip/_vendor/pygments/LICENSE
+License-File: src/pip/_vendor/pyproject_hooks/LICENSE
+License-File: src/pip/_vendor/requests/LICENSE
+License-File: src/pip/_vendor/resolvelib/LICENSE
+License-File: src/pip/_vendor/rich/LICENSE
+License-File: src/pip/_vendor/tomli/LICENSE
+License-File: src/pip/_vendor/tomli/LICENSE-HEADER
+License-File: src/pip/_vendor/tomli_w/LICENSE
+License-File: src/pip/_vendor/truststore/LICENSE
+License-File: src/pip/_vendor/urllib3/LICENSE.txt
+Dynamic: license-file
+
+pip - The Python Package Installer
+==================================
+
+.. |pypi-version| image:: https://img.shields.io/pypi/v/pip.svg
+   :target: https://pypi.org/project/pip/
+   :alt: PyPI
+
+.. |python-versions| image:: https://img.shields.io/pypi/pyversions/pip
+   :target: https://pypi.org/project/pip
+   :alt: PyPI - Python Version
+
+.. |docs-badge| image:: https://readthedocs.org/projects/pip/badge/?version=latest
+   :target: https://pip.pypa.io/en/latest
+   :alt: Documentation
+
+|pypi-version| |python-versions| |docs-badge|
+
+pip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes.
+
+Please take a look at our documentation for how to install and use pip:
+
+* `Installation`_
+* `Usage`_
+
+We release updates regularly, with a new version every 3 months. Find more details in our documentation:
+
+* `Release notes`_
+* `Release process`_
+
+If you find bugs, need help, or want to talk to the developers, please use our mailing lists or chat rooms:
+
+* `Issue tracking`_
+* `Discourse channel`_
+* `User IRC`_
+
+If you want to get involved head over to GitHub to get the source code, look at our development documentation and feel free to jump on the developer mailing lists and chat rooms:
+
+* `GitHub page`_
+* `Development documentation`_
+* `Development IRC`_
+
+Code of Conduct
+---------------
+
+Everyone interacting in the pip project's codebases, issue trackers, chat
+rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_.
+
+.. _package installer: https://packaging.python.org/guides/tool-recommendations/
+.. _Python Package Index: https://pypi.org
+.. _Installation: https://pip.pypa.io/en/stable/installation/
+.. _Usage: https://pip.pypa.io/en/stable/
+.. _Release notes: https://pip.pypa.io/en/stable/news.html
+.. _Release process: https://pip.pypa.io/en/latest/development/release-process/
+.. _GitHub page: https://github.com/pypa/pip
+.. _Development documentation: https://pip.pypa.io/en/latest/development
+.. _Issue tracking: https://github.com/pypa/pip/issues
+.. _Discourse channel: https://discuss.python.org/c/packaging
+.. _User IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa
+.. _Development IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa-dev
+.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md
diff --git a/lib/python3.1/site-packages/pip-25.2.dist-info/RECORD b/lib/python3.1/site-packages/pip-25.2.dist-info/RECORD
new file mode 100644
index 0000000000000000000000000000000000000000..32fcf98a6a04efa643c28c9540f6283753fd874a
--- /dev/null
+++ b/lib/python3.1/site-packages/pip-25.2.dist-info/RECORD
@@ -0,0 +1,861 @@
+../../../bin/pip,sha256=WuHDnaDVVUTUGBjpQBjfDSMctRl9z4aPDBH5IpIJW34,477
+../../../bin/pip3,sha256=WuHDnaDVVUTUGBjpQBjfDSMctRl9z4aPDBH5IpIJW34,477
+../../../bin/pip3.12,sha256=WuHDnaDVVUTUGBjpQBjfDSMctRl9z4aPDBH5IpIJW34,477
+pip-25.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+pip-25.2.dist-info/METADATA,sha256=l6OtFNcf2JFKOPJK9bMaH5-usFrqZqSvQNl51tetIp8,4744
+pip-25.2.dist-info/RECORD,,
+pip-25.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip-25.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pip-25.2.dist-info/direct_url.json,sha256=fT60TdWpB4XbxqHWH8gV98sjsZYl4VwhyUAbzHO8h_w,99
+pip-25.2.dist-info/entry_points.txt,sha256=eeIjuzfnfR2PrhbjnbzFU6MnSS70kZLxwaHHq6M-bD0,87
+pip-25.2.dist-info/licenses/AUTHORS.txt,sha256=ioEfMGkkizcTcIPdvjh-kYymO1E9I9dvzHjLUlKS5m8,11385
+pip-25.2.dist-info/licenses/LICENSE.txt,sha256=Y0MApmnUmurmWxLGxIySTFGkzfPR_whtw0VtyLyqIQQ,1093
+pip-25.2.dist-info/licenses/src/pip/_vendor/cachecontrol/LICENSE.txt,sha256=hu7uh74qQ_P_H1ZJb0UfaSQ5JvAl_tuwM2ZsMExMFhs,558
+pip-25.2.dist-info/licenses/src/pip/_vendor/certifi/LICENSE,sha256=6TcW2mucDVpKHfYP5pWzcPBpVgPSH2-D8FPkLPwQyvc,989
+pip-25.2.dist-info/licenses/src/pip/_vendor/dependency_groups/LICENSE.txt,sha256=GrNuPipLqGMWJThPh-ngkdsfrtA0xbIzJbMjmr8sxSU,1099
+pip-25.2.dist-info/licenses/src/pip/_vendor/distlib/LICENSE.txt,sha256=gI4QyKarjesUn_mz-xn0R6gICUYG1xKpylf-rTVSWZ0,14531
+pip-25.2.dist-info/licenses/src/pip/_vendor/distro/LICENSE,sha256=y16Ofl9KOYjhBjwULGDcLfdWBfTEZRXnduOspt-XbhQ,11325
+pip-25.2.dist-info/licenses/src/pip/_vendor/idna/LICENSE.md,sha256=pZ8LDvNjWHQQmkRhykT_enDVBpboFHZ7-vch1Mmw2w8,1541
+pip-25.2.dist-info/licenses/src/pip/_vendor/msgpack/COPYING,sha256=SS3tuoXaWHL3jmCRvNH-pHTWYNNay03ulkuKqz8AdCc,614
+pip-25.2.dist-info/licenses/src/pip/_vendor/packaging/LICENSE,sha256=ytHvW9NA1z4HS6YU0m996spceUDD2MNIUuZcSQlobEg,197
+pip-25.2.dist-info/licenses/src/pip/_vendor/packaging/LICENSE.APACHE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
+pip-25.2.dist-info/licenses/src/pip/_vendor/packaging/LICENSE.BSD,sha256=tw5-m3QvHMb5SLNMFqo5_-zpQZY2S8iP8NIYDwAo-sU,1344
+pip-25.2.dist-info/licenses/src/pip/_vendor/pkg_resources/LICENSE,sha256=htoPAa6uRjSKPD1GUZXcHOzN55956HdppkuNoEsqR0E,1023
+pip-25.2.dist-info/licenses/src/pip/_vendor/platformdirs/LICENSE,sha256=KeD9YukphQ6G6yjD_czwzv30-pSHkBHP-z0NS-1tTbY,1089
+pip-25.2.dist-info/licenses/src/pip/_vendor/pygments/LICENSE,sha256=qdZvHVJt8C4p3Oc0NtNOVuhjL0bCdbvf_HBWnogvnxc,1331
+pip-25.2.dist-info/licenses/src/pip/_vendor/pyproject_hooks/LICENSE,sha256=GyKwSbUmfW38I6Z79KhNjsBLn9-xpR02DkK0NCyLQVQ,1081
+pip-25.2.dist-info/licenses/src/pip/_vendor/requests/LICENSE,sha256=CeipvOyAZxBGUsFoaFqwkx54aPnIKEtm9a5u2uXxEws,10142
+pip-25.2.dist-info/licenses/src/pip/_vendor/resolvelib/LICENSE,sha256=84j9OMrRMRLB3A9mm76A5_hFQe26-3LzAw0sp2QsPJ0,751
+pip-25.2.dist-info/licenses/src/pip/_vendor/rich/LICENSE,sha256=3u18F6QxgVgZCj6iOcyHmlpQJxzruYrnAl9I--WNyhU,1056
+pip-25.2.dist-info/licenses/src/pip/_vendor/tomli/LICENSE,sha256=uAgWsNUwuKzLTCIReDeQmEpuO2GSLCte6S8zcqsnQv4,1072
+pip-25.2.dist-info/licenses/src/pip/_vendor/tomli/LICENSE-HEADER,sha256=l86TMJBaFy3ehw7gNh2Jvrlbo70PRUV5aqkajAGkNTE,121
+pip-25.2.dist-info/licenses/src/pip/_vendor/tomli_w/LICENSE,sha256=uAgWsNUwuKzLTCIReDeQmEpuO2GSLCte6S8zcqsnQv4,1072
+pip-25.2.dist-info/licenses/src/pip/_vendor/truststore/LICENSE,sha256=M757fo-k_Rmxdg4ajtimaL2rhSyRtpLdQUJLy3Jan8o,1086
+pip-25.2.dist-info/licenses/src/pip/_vendor/urllib3/LICENSE.txt,sha256=w3vxhuJ8-dvpYZ5V7f486nswCRzrPaY8fay-Dm13kHs,1115
+pip-25.2.dist-info/top_level.txt,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+pip/__init__.py,sha256=_lgs5Mfp0t7AGtI7sTVwxKWquz_vahWWH9kKO1cJusA,353
+pip/__main__.py,sha256=WzbhHXTbSE6gBY19mNN9m4s5o_365LOvTYSgqgbdBhE,854
+pip/__pip-runner__.py,sha256=JOoEZTwrtv7jRaXBkgSQKAE04yNyfFmGHxqpHiGHvL0,1450
+pip/__pycache__/__init__.cpython-312.pyc,,
+pip/__pycache__/__main__.cpython-312.pyc,,
+pip/__pycache__/__pip-runner__.cpython-312.pyc,,
+pip/_internal/__init__.py,sha256=S7i9Dn9aSZS0MG-2Wrve3dV9TImPzvQn5jjhp9t_uf0,511
+pip/_internal/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/__pycache__/build_env.cpython-312.pyc,,
+pip/_internal/__pycache__/cache.cpython-312.pyc,,
+pip/_internal/__pycache__/configuration.cpython-312.pyc,,
+pip/_internal/__pycache__/exceptions.cpython-312.pyc,,
+pip/_internal/__pycache__/main.cpython-312.pyc,,
+pip/_internal/__pycache__/pyproject.cpython-312.pyc,,
+pip/_internal/__pycache__/self_outdated_check.cpython-312.pyc,,
+pip/_internal/__pycache__/wheel_builder.cpython-312.pyc,,
+pip/_internal/build_env.py,sha256=5_-O5G0QtCeFKHAezqAz0IZ5_O7qCYZ-Bxddu9idpYs,11566
+pip/_internal/cache.py,sha256=rWXvsuFY7GgPERhiNp3yZHFGKdvDvkZRn2n4mXn8lAg,10364
+pip/_internal/cli/__init__.py,sha256=Iqg_tKA771XuMO1P4t_sDHnSKPzkUb9D0DqunAmw_ko,131
+pip/_internal/cli/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/cli/__pycache__/autocompletion.cpython-312.pyc,,
+pip/_internal/cli/__pycache__/base_command.cpython-312.pyc,,
+pip/_internal/cli/__pycache__/cmdoptions.cpython-312.pyc,,
+pip/_internal/cli/__pycache__/command_context.cpython-312.pyc,,
+pip/_internal/cli/__pycache__/index_command.cpython-312.pyc,,
+pip/_internal/cli/__pycache__/main.cpython-312.pyc,,
+pip/_internal/cli/__pycache__/main_parser.cpython-312.pyc,,
+pip/_internal/cli/__pycache__/parser.cpython-312.pyc,,
+pip/_internal/cli/__pycache__/progress_bars.cpython-312.pyc,,
+pip/_internal/cli/__pycache__/req_command.cpython-312.pyc,,
+pip/_internal/cli/__pycache__/spinners.cpython-312.pyc,,
+pip/_internal/cli/__pycache__/status_codes.cpython-312.pyc,,
+pip/_internal/cli/autocompletion.py,sha256=ZG2cM03nlcNrs-WG_SFTW46isx9s2Go5lUD_8-iv70o,7193
+pip/_internal/cli/base_command.py,sha256=1Nx919JRFlgURLis9XYJwtbyEEjRJa_NdHwM6iBkZvY,8716
+pip/_internal/cli/cmdoptions.py,sha256=99bW3xpXJ8Mr59OGbG7d7vN7CGPLPwi1nTmE3eMc8lE,32032
+pip/_internal/cli/command_context.py,sha256=kmu3EWZbfBega1oDamnGJTA_UaejhIQNuMj2CVmMXu0,817
+pip/_internal/cli/index_command.py,sha256=AHk6eSqboaxTXbG3v9mBrVd0dCK1MtW4w3PVudnj0WE,5717
+pip/_internal/cli/main.py,sha256=K9PtpRdg6uBrVKk8S2VZ14fAN0kP-cnA1o-FtJCN_OQ,2815
+pip/_internal/cli/main_parser.py,sha256=UugPD-hF1WtNQdow_WWduDLUH1DvElpc7EeUWjUkcNo,4329
+pip/_internal/cli/parser.py,sha256=sAOebBa5_0rRAlmBy8myMPFHQZb-bbCH1xpIJDaXaLs,10928
+pip/_internal/cli/progress_bars.py,sha256=nRTWNof-FjHfvirvECXIh7T7eAynTUVPTyHENfpbWiU,4668
+pip/_internal/cli/req_command.py,sha256=pXT_jAwcmO9PjJ1-Pl1VevBgr5pnxHsyPZUBzAfqQg8,13081
+pip/_internal/cli/spinners.py,sha256=EJzZIZNyUtJljp3-WjcsyIrqxW-HUsfWzhuW84n_Tqw,7362
+pip/_internal/cli/status_codes.py,sha256=sEFHUaUJbqv8iArL3HAtcztWZmGOFX01hTesSytDEh0,116
+pip/_internal/commands/__init__.py,sha256=aNeCbQurGWihfhQq7BqaLXHqWDQ0i3I04OS7kxK6plQ,4026
+pip/_internal/commands/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/cache.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/check.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/completion.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/configuration.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/debug.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/download.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/freeze.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/hash.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/help.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/index.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/inspect.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/install.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/list.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/lock.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/search.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/show.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/uninstall.cpython-312.pyc,,
+pip/_internal/commands/__pycache__/wheel.cpython-312.pyc,,
+pip/_internal/commands/cache.py,sha256=OrrLS6EJEha_55yPa9fTaOaonw-VpH4_lVhjxuOTChQ,8230
+pip/_internal/commands/check.py,sha256=hVFBQezQ3zj4EydoWbFQj_afPUppMt7r9JPAlY22U6Y,2244
+pip/_internal/commands/completion.py,sha256=MDwhTOBjlM4WEbOhgbhrWnlDm710i4FMjop3RBXXXCc,4530
+pip/_internal/commands/configuration.py,sha256=6gNOGrVWnOLU15zUnAiNuOMhf76RRIZvCdVD0degPRk,10105
+pip/_internal/commands/debug.py,sha256=_8IqM8Fx1_lY2STu_qspr63tufF7zyFJCyYAXtxz0N4,6805
+pip/_internal/commands/download.py,sha256=GGBSxhORV0mrad8STgORfvjRGh1qS2plvpXFNbxgcps,5249
+pip/_internal/commands/freeze.py,sha256=fxoW8AAc-bAqB_fXdNq2VnZ3JfWkFMg-bR6LcdDVO7A,3099
+pip/_internal/commands/hash.py,sha256=GO9pRN3wXC2kQaovK57TaLYBMc3IltOH92O6QEw6YE0,1679
+pip/_internal/commands/help.py,sha256=Bz3LcjNQXkz4Cu__pL4CZ86o4-HNLZj1NZWdlJhjuu0,1108
+pip/_internal/commands/index.py,sha256=8GMBVI5NvhRRHBSUq27YxDIE02DpvdJ_6qiBFgGd1co,5243
+pip/_internal/commands/inspect.py,sha256=ogm4UT7LRo8bIQcWUS1IiA25QdD4VHLa7JaPAodDttM,3177
+pip/_internal/commands/install.py,sha256=AIO-Um49e3GqMOk2SmHCt45A0W0-YhKXcpr5okjHh80,30080
+pip/_internal/commands/list.py,sha256=I4ZH604E5gpcROxEXA7eyaNEFhXx3VFVqvpscz_Ps_A,13514
+pip/_internal/commands/lock.py,sha256=SxXGnU2EH-TG-fs9fuZHAuMOH15Lzy1K7e_KJ4vtSr0,5917
+pip/_internal/commands/search.py,sha256=zbMsX_YASj6kXA6XIBgTDv0bGK51xG-CV3IynZJcE-c,5782
+pip/_internal/commands/show.py,sha256=oLVJIfKWmDKm0SsQGEi3pozNiqrXjTras_fbBSYKpBA,8066
+pip/_internal/commands/uninstall.py,sha256=CsOihqvb6ZA6O67L70oXeoLHeOfNzMM88H9g-9aocgw,3868
+pip/_internal/commands/wheel.py,sha256=vcNgnhxwilocRQJEBdWQvJ8qvO0RfWmz9LwrUBadvYE,6322
+pip/_internal/configuration.py,sha256=BomQ3pC84J6zCXDQpND_OmFY7mubE9cMBZVohNUzMvY,14587
+pip/_internal/distributions/__init__.py,sha256=Hq6kt6gXBgjNit5hTTWLAzeCNOKoB-N0pGYSqehrli8,858
+pip/_internal/distributions/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/distributions/__pycache__/base.cpython-312.pyc,,
+pip/_internal/distributions/__pycache__/installed.cpython-312.pyc,,
+pip/_internal/distributions/__pycache__/sdist.cpython-312.pyc,,
+pip/_internal/distributions/__pycache__/wheel.cpython-312.pyc,,
+pip/_internal/distributions/base.py,sha256=l-OTCAIs25lsapejA6IYpPZxSM5-BET4sdZDkql8jiY,1830
+pip/_internal/distributions/installed.py,sha256=kgIEE_1NzjZxLBSC-v5s64uOFZlVEt3aPrjTtL6x2XY,929
+pip/_internal/distributions/sdist.py,sha256=gYgrzI0lstHJRKweOdL_m6LtqDxtfuo_yCL9HjmH-xw,6952
+pip/_internal/distributions/wheel.py,sha256=_HbG0OehF8dwj4UX-xV__tXLwgPus9OjMEf2NTRqBbE,1364
+pip/_internal/exceptions.py,sha256=bRTEn6Jlw-vxHjixtiu0K79jXJu0X5FVC0L62Bdb0KI,28974
+pip/_internal/index/__init__.py,sha256=tzwMH_fhQeubwMqHdSivasg1cRgTSbNg2CiMVnzMmyU,29
+pip/_internal/index/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/index/__pycache__/collector.cpython-312.pyc,,
+pip/_internal/index/__pycache__/package_finder.cpython-312.pyc,,
+pip/_internal/index/__pycache__/sources.cpython-312.pyc,,
+pip/_internal/index/collector.py,sha256=PCB3thVWRiSBowGtpv1elIPFc-GvEqhZiNgZD7b0vBc,16185
+pip/_internal/index/package_finder.py,sha256=Kd2FmWJFAcEg2Sy2pTGQX8WPFC8B9U_3b1VK7i8sx9o,38827
+pip/_internal/index/sources.py,sha256=nXJkOjhLy-O2FsrKU9RIqCOqgY2PsoKWybtZjjRgqU0,8639
+pip/_internal/locations/__init__.py,sha256=2SADX0Gr9BIpx19AO7Feq89nOmBQGEbl1IWjBpnaE9E,14185
+pip/_internal/locations/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/locations/__pycache__/_distutils.cpython-312.pyc,,
+pip/_internal/locations/__pycache__/_sysconfig.cpython-312.pyc,,
+pip/_internal/locations/__pycache__/base.cpython-312.pyc,,
+pip/_internal/locations/_distutils.py,sha256=jpFj4V00rD9IR3vA9TqrGkwcdNVFc58LsChZavge9JY,5975
+pip/_internal/locations/_sysconfig.py,sha256=NhcEi1_25w9cTTcH4RyOjD4UHW6Ijks0uKy1PL1_j_8,7716
+pip/_internal/locations/base.py,sha256=AImjYJWxOtDkc0KKc6Y4Gz677cg91caMA4L94B9FZEg,2550
+pip/_internal/main.py,sha256=1cHqjsfFCrMFf3B5twzocxTJUdHMLoXUpy5lJoFqUi8,338
+pip/_internal/metadata/__init__.py,sha256=a19B00IsX25khBt8oWWy6_kKgjCM4zeh-FqYteCOFwA,5714
+pip/_internal/metadata/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/metadata/__pycache__/_json.cpython-312.pyc,,
+pip/_internal/metadata/__pycache__/base.cpython-312.pyc,,
+pip/_internal/metadata/__pycache__/pkg_resources.cpython-312.pyc,,
+pip/_internal/metadata/_json.py,sha256=hNvnMHOXLAyNlzirWhPL9Nx2CvCqa1iRma6Osq1YfV8,2711
+pip/_internal/metadata/base.py,sha256=BGuMenlcQT8i7j9iclrfdC3vSwgvhr8gjn955cCy16s,25420
+pip/_internal/metadata/importlib/__init__.py,sha256=jUUidoxnHcfITHHaAWG1G2i5fdBYklv_uJcjo2x7VYE,135
+pip/_internal/metadata/importlib/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/metadata/importlib/__pycache__/_compat.cpython-312.pyc,,
+pip/_internal/metadata/importlib/__pycache__/_dists.cpython-312.pyc,,
+pip/_internal/metadata/importlib/__pycache__/_envs.cpython-312.pyc,,
+pip/_internal/metadata/importlib/_compat.py,sha256=sneVh4_6WxQZK4ljdl3ylVuP-q0ttSqbgl9mWt0HnOg,2804
+pip/_internal/metadata/importlib/_dists.py,sha256=aOUgCX_AtA8B-lWLu-HfXA-ivMJWNxHPAohVcpQj2g4,8259
+pip/_internal/metadata/importlib/_envs.py,sha256=H3qVLXVh4LWvrPvu_ekXf3dfbtwnlhNJQP2pxXpccfU,5333
+pip/_internal/metadata/pkg_resources.py,sha256=NO76ZrfR2-LKJTyaXrmQoGhmJMArALvacrlZHViSDT8,10544
+pip/_internal/models/__init__.py,sha256=AjmCEBxX_MH9f_jVjIGNCFJKYCYeSEe18yyvNx4uRKQ,62
+pip/_internal/models/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/models/__pycache__/candidate.cpython-312.pyc,,
+pip/_internal/models/__pycache__/direct_url.cpython-312.pyc,,
+pip/_internal/models/__pycache__/format_control.cpython-312.pyc,,
+pip/_internal/models/__pycache__/index.cpython-312.pyc,,
+pip/_internal/models/__pycache__/installation_report.cpython-312.pyc,,
+pip/_internal/models/__pycache__/link.cpython-312.pyc,,
+pip/_internal/models/__pycache__/pylock.cpython-312.pyc,,
+pip/_internal/models/__pycache__/scheme.cpython-312.pyc,,
+pip/_internal/models/__pycache__/search_scope.cpython-312.pyc,,
+pip/_internal/models/__pycache__/selection_prefs.cpython-312.pyc,,
+pip/_internal/models/__pycache__/target_python.cpython-312.pyc,,
+pip/_internal/models/__pycache__/wheel.cpython-312.pyc,,
+pip/_internal/models/candidate.py,sha256=zzgFRuw_kWPjKpGw7LC0ZUMD2CQ2EberUIYs8izjdCA,753
+pip/_internal/models/direct_url.py,sha256=4NMWacu_QzPPWREC1te7v6Wfv-2HkI4tvSJF-CBgLh4,6555
+pip/_internal/models/format_control.py,sha256=PwemYG1L27BM0f1KP61rm24wShENFyxqlD1TWu34alc,2471
+pip/_internal/models/index.py,sha256=tYnL8oxGi4aSNWur0mG8DAP7rC6yuha_MwJO8xw0crI,1030
+pip/_internal/models/installation_report.py,sha256=cqfWJ93ThCxjcacqSWryOCD2XtIn1CZrgzZxAv5FQZ0,2839
+pip/_internal/models/link.py,sha256=h4lI8MaDA7DTIxIJ_NgDE64EE4h1sX3AB23Y1Qu8W-k,21793
+pip/_internal/models/pylock.py,sha256=Vmaa71gOSV0ZYzRgWiIm4KwVbClaahMcuvKCkP_ZznA,6211
+pip/_internal/models/scheme.py,sha256=PakmHJM3e8OOWSZFtfz1Az7f1meONJnkGuQxFlt3wBE,575
+pip/_internal/models/search_scope.py,sha256=1hxU2IVsAaLZVjp0CbzJbYaYzCxv72_Qbg3JL0qhXo0,4507
+pip/_internal/models/selection_prefs.py,sha256=lgYyo4W8lb22wsYx2ElBBB0cvSNlBVgucwBzL43dfzE,2016
+pip/_internal/models/target_python.py,sha256=I0eFS-eia3kwhrOvgsphFZtNAB2IwXZ9Sr9fp6IjBP4,4243
+pip/_internal/models/wheel.py,sha256=xWO0K-YanSL3OfMZUN1gHlEUV0YJOVYMQ4wrwmA9Zis,5526
+pip/_internal/network/__init__.py,sha256=FMy06P__y6jMjUc8z3ZcQdKF-pmZ2zM14_vBeHPGhUI,49
+pip/_internal/network/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/network/__pycache__/auth.cpython-312.pyc,,
+pip/_internal/network/__pycache__/cache.cpython-312.pyc,,
+pip/_internal/network/__pycache__/download.cpython-312.pyc,,
+pip/_internal/network/__pycache__/lazy_wheel.cpython-312.pyc,,
+pip/_internal/network/__pycache__/session.cpython-312.pyc,,
+pip/_internal/network/__pycache__/utils.cpython-312.pyc,,
+pip/_internal/network/__pycache__/xmlrpc.cpython-312.pyc,,
+pip/_internal/network/auth.py,sha256=uAwRGAYnVtgNSZm4HMC3BMACkgA7ku4m8wupiX6LpK8,20681
+pip/_internal/network/cache.py,sha256=u5LtQbnCzMU6vFT0eUUcYe6iKiwALMqrn1jUZj4CqaY,5302
+pip/_internal/network/download.py,sha256=HgsFvTkPDdgg0zUehose_J-542-9R0FpyipRw5BhxAM,12682
+pip/_internal/network/lazy_wheel.py,sha256=5H7_s-_AK6br1K5NFcyUsiocK9E6vwrJY5kzwjMv0EM,7651
+pip/_internal/network/session.py,sha256=eE-VUIJGU9YeeaVy7tVAvMRWigMsyuAMpxkjlbptbjo,19188
+pip/_internal/network/utils.py,sha256=ACsXd1msqNCidHVXsu7LHUSr8NgaypcOKQ4KG-Z_wJM,4091
+pip/_internal/network/xmlrpc.py,sha256=_-Rnk3vOff8uF9hAGmT6SLALflY1gMBcbGwS12fb_Y4,1830
+pip/_internal/operations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_internal/operations/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/operations/__pycache__/check.cpython-312.pyc,,
+pip/_internal/operations/__pycache__/freeze.cpython-312.pyc,,
+pip/_internal/operations/__pycache__/prepare.cpython-312.pyc,,
+pip/_internal/operations/build/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_internal/operations/build/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/operations/build/__pycache__/build_tracker.cpython-312.pyc,,
+pip/_internal/operations/build/__pycache__/metadata.cpython-312.pyc,,
+pip/_internal/operations/build/__pycache__/metadata_editable.cpython-312.pyc,,
+pip/_internal/operations/build/__pycache__/metadata_legacy.cpython-312.pyc,,
+pip/_internal/operations/build/__pycache__/wheel.cpython-312.pyc,,
+pip/_internal/operations/build/__pycache__/wheel_editable.cpython-312.pyc,,
+pip/_internal/operations/build/__pycache__/wheel_legacy.cpython-312.pyc,,
+pip/_internal/operations/build/build_tracker.py,sha256=W3b5cmkMWPaE6QIwfzsTayJo7-OlxFHWDxfPuax1KcE,4771
+pip/_internal/operations/build/metadata.py,sha256=INHaeiRfOiLYCXApfDNRo9Cw2xI4VwTc0KItvfdfOjk,1421
+pip/_internal/operations/build/metadata_editable.py,sha256=oWudMsnjy4loO_Jy7g4N9nxsnaEX_iDlVRgCy7pu1rs,1509
+pip/_internal/operations/build/metadata_legacy.py,sha256=wv8cFA0wTqF62Jlm9QwloYZsofOyQ7sWBBmvCcVvn1k,2189
+pip/_internal/operations/build/wheel.py,sha256=toBJE5atKJGsSckY5ztVAQpRss25f049f-nJaiJsiD8,1080
+pip/_internal/operations/build/wheel_editable.py,sha256=Djx7hZqhejgYUcdQD5p4mn7AVFN3VQ7bOSs7b90TtOI,1422
+pip/_internal/operations/build/wheel_legacy.py,sha256=LaHpG4_s6415iLjuTb6seO3JIJ7097yQUIthcmpqaOY,3616
+pip/_internal/operations/check.py,sha256=yC2XWth6iehGGE_fj7XRJLjVKBsTIG3ZoWRkFi3rOwc,5894
+pip/_internal/operations/freeze.py,sha256=PDdY-y_ZtZZJLAKcaWPIGRKAGW7DXR48f0aMRU0j7BA,9854
+pip/_internal/operations/install/__init__.py,sha256=ak-UETcQPKlFZaWoYKWu5QVXbpFBvg0sXc3i0O4vSYY,50
+pip/_internal/operations/install/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/operations/install/__pycache__/editable_legacy.cpython-312.pyc,,
+pip/_internal/operations/install/__pycache__/wheel.cpython-312.pyc,,
+pip/_internal/operations/install/editable_legacy.py,sha256=zS6Hm-9SHcrVuA_9Irc9Qc8CIoFLDMZiSvi5rizZe34,1311
+pip/_internal/operations/install/wheel.py,sha256=8aepxxAFmnzZFtcMCv-1I4T_maEkQd4hXZztYWE4yR0,27956
+pip/_internal/operations/prepare.py,sha256=gdbQUAjxLqqz8P2KFem2OEJ_6V3aTuozQkiBLfCCVAU,28613
+pip/_internal/pyproject.py,sha256=T72qz0buaY0s-mgrXLUU1sONf4Rk97dJu0qpniBCGU8,7233
+pip/_internal/req/__init__.py,sha256=vM0bWAl3By5ALRIXWmKdkMca18KQfbGXVJZm3Igpwmg,3122
+pip/_internal/req/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/req/__pycache__/constructors.cpython-312.pyc,,
+pip/_internal/req/__pycache__/req_dependency_group.cpython-312.pyc,,
+pip/_internal/req/__pycache__/req_file.cpython-312.pyc,,
+pip/_internal/req/__pycache__/req_install.cpython-312.pyc,,
+pip/_internal/req/__pycache__/req_set.cpython-312.pyc,,
+pip/_internal/req/__pycache__/req_uninstall.cpython-312.pyc,,
+pip/_internal/req/constructors.py,sha256=QlLjBIg5xPxyQx_hGGB7Fz527ruAXC6nbVMoGfXzNnM,18320
+pip/_internal/req/req_dependency_group.py,sha256=0yEQCUaO5Bza66Y3D5o9JRf0qII5QgCRugn1x5aRivA,2618
+pip/_internal/req/req_file.py,sha256=j_1PcBDO0aKb8pjjRMsZQrwts5YNrKsrHjPyV56XoHo,20161
+pip/_internal/req/req_install.py,sha256=N4LxV9LHimgle5zEkNFk9WSfi51KXvK9y73mvYl9uig,35718
+pip/_internal/req/req_set.py,sha256=awkqIXnYA4Prmsj0Qb3zhqdbYUmXd-1o0P-KZ3mvRQs,2828
+pip/_internal/req/req_uninstall.py,sha256=dCmOHt-9RaJBq921L4tMH3PmIBDetGplnbjRKXmGt00,24099
+pip/_internal/resolution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_internal/resolution/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/resolution/__pycache__/base.cpython-312.pyc,,
+pip/_internal/resolution/base.py,sha256=RIsqSP79olPdOgtPKW-oOQ364ICVopehA6RfGkRfe2s,577
+pip/_internal/resolution/legacy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_internal/resolution/legacy/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/resolution/legacy/__pycache__/resolver.cpython-312.pyc,,
+pip/_internal/resolution/legacy/resolver.py,sha256=bwUqE66etz2bcPabqxed18-iyqqb-kx3Er2aT6GeUJY,24060
+pip/_internal/resolution/resolvelib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_internal/resolution/resolvelib/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/resolution/resolvelib/__pycache__/base.cpython-312.pyc,,
+pip/_internal/resolution/resolvelib/__pycache__/candidates.cpython-312.pyc,,
+pip/_internal/resolution/resolvelib/__pycache__/factory.cpython-312.pyc,,
+pip/_internal/resolution/resolvelib/__pycache__/found_candidates.cpython-312.pyc,,
+pip/_internal/resolution/resolvelib/__pycache__/provider.cpython-312.pyc,,
+pip/_internal/resolution/resolvelib/__pycache__/reporter.cpython-312.pyc,,
+pip/_internal/resolution/resolvelib/__pycache__/requirements.cpython-312.pyc,,
+pip/_internal/resolution/resolvelib/__pycache__/resolver.cpython-312.pyc,,
+pip/_internal/resolution/resolvelib/base.py,sha256=_AoP0ZWlaSct8CRDn2ol3CbNn4zDtnh_0zQGjXASDKI,5047
+pip/_internal/resolution/resolvelib/candidates.py,sha256=dblosDcMumdT075Y8D1yXiJrWF_wDgIGe_8hUQIneTw,20208
+pip/_internal/resolution/resolvelib/factory.py,sha256=XdCpLq59oVIHMMekc2cgD7imcVD00_ioTnu9_k3iq3E,32577
+pip/_internal/resolution/resolvelib/found_candidates.py,sha256=8bZYDCZLXSdLHy_s1o5f4r15HmKvqFUhzBUQOF21Lr4,6018
+pip/_internal/resolution/resolvelib/provider.py,sha256=1TJC2o7F02aTMaoqa0Ayz45PrE-pQ5XshgGUmweESkk,11144
+pip/_internal/resolution/resolvelib/reporter.py,sha256=atiTh1bnKc-KfG841nRw3dk0WbOhr_L8luJHcY0JoSs,3270
+pip/_internal/resolution/resolvelib/requirements.py,sha256=z0gXmWfo03ynOnhF8kpj5SycgroerDhQV0VWzmAKAfg,8076
+pip/_internal/resolution/resolvelib/resolver.py,sha256=f0fmYzB9G4wWxssfj4qerDi_F_8vjAaYPbGLb7N7tQw,13617
+pip/_internal/self_outdated_check.py,sha256=wfikZmcGVIrnepL413-0rnQ3jeOA0unOKPle0Ovwoho,8326
+pip/_internal/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_internal/utils/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/_jaraco_text.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/_log.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/appdirs.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/compat.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/compatibility_tags.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/datetime.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/deprecation.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/direct_url_helpers.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/egg_link.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/entrypoints.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/filesystem.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/filetypes.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/glibc.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/hashes.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/logging.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/misc.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/packaging.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/retry.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/setuptools_build.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/subprocess.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/temp_dir.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/unpacking.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/urls.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/virtualenv.cpython-312.pyc,,
+pip/_internal/utils/__pycache__/wheel.cpython-312.pyc,,
+pip/_internal/utils/_jaraco_text.py,sha256=M15uUPIh5NpP1tdUGBxRau6q1ZAEtI8-XyLEETscFfE,3350
+pip/_internal/utils/_log.py,sha256=-jHLOE_THaZz5BFcCnoSL9EYAtJ0nXem49s9of4jvKw,1015
+pip/_internal/utils/appdirs.py,sha256=LrzDPZMKVh0rubtCx9vu3XlZbLCSug6VSj4Qsvt66BA,1681
+pip/_internal/utils/compat.py,sha256=C9LHXJAKkwAH8Hn3nPkz9EYK3rqPBeO_IXkOG2zzsdQ,2514
+pip/_internal/utils/compatibility_tags.py,sha256=DiNSLqpuruXUamGQwOJ2WZByDGLTGaXi9O-Xf8fOi34,6630
+pip/_internal/utils/datetime.py,sha256=Gt29Ml4ToPSM88j54iu43WKtrU9A-moP4QmMiiqzedU,241
+pip/_internal/utils/deprecation.py,sha256=HVhvyO5qiRFcG88PhZlp_87qdKQNwPTUIIHWtsTR2yI,3696
+pip/_internal/utils/direct_url_helpers.py,sha256=ttKv4GMUqlRwPPog9_CUopy6SDgoxVILzeBJzgfn2tg,3200
+pip/_internal/utils/egg_link.py,sha256=YWfsrbmfcrfWgqQYy6OuIjsyb9IfL1q_2v4zsms1WjI,2459
+pip/_internal/utils/entrypoints.py,sha256=uPjAyShKObdotjQjJUzprQ6r3xQvDIZwUYfHHqZ7Dok,3324
+pip/_internal/utils/filesystem.py,sha256=du4lOOlTOBq22V5kf0yXMydo1KU2Cs-cMtYs3bEk13c,4988
+pip/_internal/utils/filetypes.py,sha256=sEMa38qaqjvx1Zid3OCAUja31BOBU-USuSMPBvU3yjo,689
+pip/_internal/utils/glibc.py,sha256=sEh8RJJLYSdRvTqAO4THVPPA-YSDVLD4SI9So-bxX1U,3726
+pip/_internal/utils/hashes.py,sha256=d32UI1en8nyqZzdZQvxUVdfeBoe4ADWx7HtrIM4-XQ4,4998
+pip/_internal/utils/logging.py,sha256=RtRe7Vp0COC4UBewYdfKicXjCTmHXpDZHdReTzJvB78,12108
+pip/_internal/utils/misc.py,sha256=1jEpqjfqYmQ6K3D4_O8xXSPn8aEfH2uMOlNM7KPvSrg,23374
+pip/_internal/utils/packaging.py,sha256=s5tpUmFumwV0H9JSTzryrIY4JwQM8paGt7Sm7eNwt2Y,1601
+pip/_internal/utils/retry.py,sha256=83wReEB2rcntMZ5VLd7ascaYSjn_kLdlQCqxILxWkPM,1461
+pip/_internal/utils/setuptools_build.py,sha256=0RK4K07qpaBgnYm50_f5d5VdedYPk1fl9QNKaOa60XM,4499
+pip/_internal/utils/subprocess.py,sha256=r4-Ba_Yc3uZXQpi0K4pZFsCT_QqdSvtF3XJ-204QWaA,8983
+pip/_internal/utils/temp_dir.py,sha256=D9c8D7WOProOO8GGDqpBeVSj10NGFmunG0o2TodjjIU,9307
+pip/_internal/utils/unpacking.py,sha256=4tY2D-sbtBt2Lcws98Em3MPAPdHnXSBXnpSU0LoGUZQ,11939
+pip/_internal/utils/urls.py,sha256=aF_eg9ul5d8bMCxfSSSxQcfs-OpJdbStYqZHoy2K1RE,1601
+pip/_internal/utils/virtualenv.py,sha256=mX-UPyw1MPxhwUxKhbqWWX70J6PHXAJjVVrRnG0h9mc,3455
+pip/_internal/utils/wheel.py,sha256=YdRuj6MicG-Q9Mg03FbUv1WTLam6Lc7AgijY4voVyis,4468
+pip/_internal/vcs/__init__.py,sha256=UAqvzpbi0VbZo3Ub6skEeZAw-ooIZR-zX_WpCbxyCoU,596
+pip/_internal/vcs/__pycache__/__init__.cpython-312.pyc,,
+pip/_internal/vcs/__pycache__/bazaar.cpython-312.pyc,,
+pip/_internal/vcs/__pycache__/git.cpython-312.pyc,,
+pip/_internal/vcs/__pycache__/mercurial.cpython-312.pyc,,
+pip/_internal/vcs/__pycache__/subversion.cpython-312.pyc,,
+pip/_internal/vcs/__pycache__/versioncontrol.cpython-312.pyc,,
+pip/_internal/vcs/bazaar.py,sha256=3W1eHjkYx2vc6boeb2NBh4I_rlGAXM-vrzfNhLm1Rxg,3734
+pip/_internal/vcs/git.py,sha256=TTeqDuzS-_BFSNuUStVWmE2nGDpKuvUhBBJk_CCQXV0,19144
+pip/_internal/vcs/mercurial.py,sha256=w1ZJWLKqNP1onEjkfjlwBVnMqPZNSIER8ayjQcnTq4w,5575
+pip/_internal/vcs/subversion.py,sha256=uUgdPvxmvEB8Qwtjr0Hc0XgFjbiNi5cbvI4vARLOJXo,11787
+pip/_internal/vcs/versioncontrol.py,sha256=d-v1mcLxofg2FaIqBrV-e-ZcjOgQhS0oxXpki1v1yXs,22502
+pip/_internal/wheel_builder.py,sha256=PDF2w6pxeec5skP3gLIrR_VrJrAO4pHRpUnkNbJ8R5Q,11225
+pip/_vendor/__init__.py,sha256=WzusPTGWIMeQQWSVJ0h2rafGkVTa9WKJ2HT-2-EoZrU,4907
+pip/_vendor/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/cachecontrol/__init__.py,sha256=BF2n5OeQz1QW2xSey2LxfNCtwbjnTadXdIH2toqJecg,677
+pip/_vendor/cachecontrol/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/cachecontrol/__pycache__/_cmd.cpython-312.pyc,,
+pip/_vendor/cachecontrol/__pycache__/adapter.cpython-312.pyc,,
+pip/_vendor/cachecontrol/__pycache__/cache.cpython-312.pyc,,
+pip/_vendor/cachecontrol/__pycache__/controller.cpython-312.pyc,,
+pip/_vendor/cachecontrol/__pycache__/filewrapper.cpython-312.pyc,,
+pip/_vendor/cachecontrol/__pycache__/heuristics.cpython-312.pyc,,
+pip/_vendor/cachecontrol/__pycache__/serialize.cpython-312.pyc,,
+pip/_vendor/cachecontrol/__pycache__/wrapper.cpython-312.pyc,,
+pip/_vendor/cachecontrol/_cmd.py,sha256=iist2EpzJvDVIhMAxXq8iFnTBsiZAd6iplxfmNboNyk,1737
+pip/_vendor/cachecontrol/adapter.py,sha256=8y6rTPXOzVHmDKCW5CR9sivLVuDv-cpdGcZYdRWNaPw,6599
+pip/_vendor/cachecontrol/cache.py,sha256=OXwv7Fn2AwnKNiahJHnjtvaKLndvVLv_-zO-ltlV9qI,1953
+pip/_vendor/cachecontrol/caches/__init__.py,sha256=dtrrroK5BnADR1GWjCZ19aZ0tFsMfvFBtLQQU1sp_ag,303
+pip/_vendor/cachecontrol/caches/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/cachecontrol/caches/__pycache__/file_cache.cpython-312.pyc,,
+pip/_vendor/cachecontrol/caches/__pycache__/redis_cache.cpython-312.pyc,,
+pip/_vendor/cachecontrol/caches/file_cache.py,sha256=d8upFmy_zwaCmlbWEVBlLXFddt8Zw8c5SFpxeOZsdfw,4117
+pip/_vendor/cachecontrol/caches/redis_cache.py,sha256=9rmqwtYu_ljVkW6_oLqbC7EaX_a8YT_yLuna-eS0dgo,1386
+pip/_vendor/cachecontrol/controller.py,sha256=cx0Hl8xLZgUuXuy78Gih9AYjCtqurmYjVJxyA4yWt7w,19101
+pip/_vendor/cachecontrol/filewrapper.py,sha256=2ktXNPE0KqnyzF24aOsKCA58HQq1xeC6l2g6_zwjghc,4291
+pip/_vendor/cachecontrol/heuristics.py,sha256=gqMXU8w0gQuEQiSdu3Yg-0vd9kW7nrWKbLca75rheGE,4881
+pip/_vendor/cachecontrol/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/cachecontrol/serialize.py,sha256=HQd2IllQ05HzPkVLMXTF2uX5mjEQjDBkxCqUJUODpZk,5163
+pip/_vendor/cachecontrol/wrapper.py,sha256=hsGc7g8QGQTT-4f8tgz3AM5qwScg6FO0BSdLSRdEvpU,1417
+pip/_vendor/certifi/__init__.py,sha256=dvNDUdAXp-hzoYcf09PXzLmHIlPfypaBQPFp5esDXyo,94
+pip/_vendor/certifi/__main__.py,sha256=1k3Cr95vCxxGRGDljrW3wMdpZdL3Nhf0u1n-k2qdsCY,255
+pip/_vendor/certifi/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/certifi/__pycache__/__main__.cpython-312.pyc,,
+pip/_vendor/certifi/__pycache__/core.cpython-312.pyc,,
+pip/_vendor/certifi/cacert.pem,sha256=lm3cYJxEv9SoAx4Atc3NiT1D92d965vcID6H39f_93o,290057
+pip/_vendor/certifi/core.py,sha256=gu_ECVI1m3Rq0ytpsNE61hgQGcKaOAt9Rs9G8KsTCOI,3442
+pip/_vendor/certifi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/dependency_groups/__init__.py,sha256=C3OFu0NGwDzQ4LOmmSOFPsRSvkbBn-mdd4j_5YqJw-s,250
+pip/_vendor/dependency_groups/__main__.py,sha256=UNTM7P5mfVtT7wDi9kOTXWgV3fu3e8bTrt1Qp1jvjKo,1709
+pip/_vendor/dependency_groups/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/dependency_groups/__pycache__/__main__.cpython-312.pyc,,
+pip/_vendor/dependency_groups/__pycache__/_implementation.cpython-312.pyc,,
+pip/_vendor/dependency_groups/__pycache__/_lint_dependency_groups.cpython-312.pyc,,
+pip/_vendor/dependency_groups/__pycache__/_pip_wrapper.cpython-312.pyc,,
+pip/_vendor/dependency_groups/__pycache__/_toml_compat.cpython-312.pyc,,
+pip/_vendor/dependency_groups/_implementation.py,sha256=Gqb2DlQELRakeHlKf6QtQSW0M-bcEomxHw4JsvID1ls,8041
+pip/_vendor/dependency_groups/_lint_dependency_groups.py,sha256=yp-DDqKXtbkDTNa0ifa-FmOA8ra24lPZEXftW-R5AuI,1710
+pip/_vendor/dependency_groups/_pip_wrapper.py,sha256=nuVW_w_ntVxpE26ELEvngMY0N04sFLsijXRyZZROFG8,1865
+pip/_vendor/dependency_groups/_toml_compat.py,sha256=BHnXnFacm3DeolsA35GjI6qkDApvua-1F20kv3BfZWE,285
+pip/_vendor/dependency_groups/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/distlib/__init__.py,sha256=Deo3uo98aUyIfdKJNqofeSEFWwDzrV2QeGLXLsgq0Ag,625
+pip/_vendor/distlib/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/distlib/__pycache__/compat.cpython-312.pyc,,
+pip/_vendor/distlib/__pycache__/resources.cpython-312.pyc,,
+pip/_vendor/distlib/__pycache__/scripts.cpython-312.pyc,,
+pip/_vendor/distlib/__pycache__/util.cpython-312.pyc,,
+pip/_vendor/distlib/compat.py,sha256=2jRSjRI4o-vlXeTK2BCGIUhkc6e9ZGhSsacRM5oseTw,41467
+pip/_vendor/distlib/resources.py,sha256=LwbPksc0A1JMbi6XnuPdMBUn83X7BPuFNWqPGEKI698,10820
+pip/_vendor/distlib/scripts.py,sha256=Qvp76E9Jc3IgyYubnpqI9fS7eseGOe4FjpeVKqKt9Iw,18612
+pip/_vendor/distlib/t32.exe,sha256=a0GV5kCoWsMutvliiCKmIgV98eRZ33wXoS-XrqvJQVs,97792
+pip/_vendor/distlib/t64-arm.exe,sha256=68TAa32V504xVBnufojh0PcenpR3U4wAqTqf-MZqbPw,182784
+pip/_vendor/distlib/t64.exe,sha256=gaYY8hy4fbkHYTTnA4i26ct8IQZzkBG2pRdy0iyuBrc,108032
+pip/_vendor/distlib/util.py,sha256=vMPGvsS4j9hF6Y9k3Tyom1aaHLb0rFmZAEyzeAdel9w,66682
+pip/_vendor/distlib/w32.exe,sha256=R4csx3-OGM9kL4aPIzQKRo5TfmRSHZo6QWyLhDhNBks,91648
+pip/_vendor/distlib/w64-arm.exe,sha256=xdyYhKj0WDcVUOCb05blQYvzdYIKMbmJn2SZvzkcey4,168448
+pip/_vendor/distlib/w64.exe,sha256=ejGf-rojoBfXseGLpya6bFTFPWRG21X5KvU8J5iU-K0,101888
+pip/_vendor/distro/__init__.py,sha256=2fHjF-SfgPvjyNZ1iHh_wjqWdR_Yo5ODHwZC0jLBPhc,981
+pip/_vendor/distro/__main__.py,sha256=bu9d3TifoKciZFcqRBuygV3GSuThnVD_m2IK4cz96Vs,64
+pip/_vendor/distro/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/distro/__pycache__/__main__.cpython-312.pyc,,
+pip/_vendor/distro/__pycache__/distro.cpython-312.pyc,,
+pip/_vendor/distro/distro.py,sha256=XqbefacAhDT4zr_trnbA15eY8vdK4GTghgmvUGrEM_4,49430
+pip/_vendor/distro/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/idna/__init__.py,sha256=MPqNDLZbXqGaNdXxAFhiqFPKEQXju2jNQhCey6-5eJM,868
+pip/_vendor/idna/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/idna/__pycache__/codec.cpython-312.pyc,,
+pip/_vendor/idna/__pycache__/compat.cpython-312.pyc,,
+pip/_vendor/idna/__pycache__/core.cpython-312.pyc,,
+pip/_vendor/idna/__pycache__/idnadata.cpython-312.pyc,,
+pip/_vendor/idna/__pycache__/intranges.cpython-312.pyc,,
+pip/_vendor/idna/__pycache__/package_data.cpython-312.pyc,,
+pip/_vendor/idna/__pycache__/uts46data.cpython-312.pyc,,
+pip/_vendor/idna/codec.py,sha256=PEew3ItwzjW4hymbasnty2N2OXvNcgHB-JjrBuxHPYY,3422
+pip/_vendor/idna/compat.py,sha256=RzLy6QQCdl9784aFhb2EX9EKGCJjg0P3PilGdeXXcx8,316
+pip/_vendor/idna/core.py,sha256=YJYyAMnwiQEPjVC4-Fqu_p4CJ6yKKuDGmppBNQNQpFs,13239
+pip/_vendor/idna/idnadata.py,sha256=W30GcIGvtOWYwAjZj4ZjuouUutC6ffgNuyjJy7fZ-lo,78306
+pip/_vendor/idna/intranges.py,sha256=amUtkdhYcQG8Zr-CoMM_kVRacxkivC1WgxN1b63KKdU,1898
+pip/_vendor/idna/package_data.py,sha256=q59S3OXsc5VI8j6vSD0sGBMyk6zZ4vWFREE88yCJYKs,21
+pip/_vendor/idna/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/idna/uts46data.py,sha256=rt90K9J40gUSwppDPCrhjgi5AA6pWM65dEGRSf6rIhM,239289
+pip/_vendor/msgpack/__init__.py,sha256=q2T5PRsITVpeytgS0WiSqok9IY8V_aFiC8VVlXTCTgA,1109
+pip/_vendor/msgpack/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/msgpack/__pycache__/exceptions.cpython-312.pyc,,
+pip/_vendor/msgpack/__pycache__/ext.cpython-312.pyc,,
+pip/_vendor/msgpack/__pycache__/fallback.cpython-312.pyc,,
+pip/_vendor/msgpack/exceptions.py,sha256=dCTWei8dpkrMsQDcjQk74ATl9HsIBH0ybt8zOPNqMYc,1081
+pip/_vendor/msgpack/ext.py,sha256=kteJv03n9tYzd5oo3xYopVTo4vRaAxonBQQJhXohZZo,5726
+pip/_vendor/msgpack/fallback.py,sha256=0g1Pzp0vtmBEmJ5w9F3s_-JMVURP8RS4G1cc5TRaAsI,32390
+pip/_vendor/packaging/__init__.py,sha256=_0cDiPVf2S-bNfVmZguxxzmrIYWlyASxpqph4qsJWUc,494
+pip/_vendor/packaging/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/packaging/__pycache__/_elffile.cpython-312.pyc,,
+pip/_vendor/packaging/__pycache__/_manylinux.cpython-312.pyc,,
+pip/_vendor/packaging/__pycache__/_musllinux.cpython-312.pyc,,
+pip/_vendor/packaging/__pycache__/_parser.cpython-312.pyc,,
+pip/_vendor/packaging/__pycache__/_structures.cpython-312.pyc,,
+pip/_vendor/packaging/__pycache__/_tokenizer.cpython-312.pyc,,
+pip/_vendor/packaging/__pycache__/markers.cpython-312.pyc,,
+pip/_vendor/packaging/__pycache__/metadata.cpython-312.pyc,,
+pip/_vendor/packaging/__pycache__/requirements.cpython-312.pyc,,
+pip/_vendor/packaging/__pycache__/specifiers.cpython-312.pyc,,
+pip/_vendor/packaging/__pycache__/tags.cpython-312.pyc,,
+pip/_vendor/packaging/__pycache__/utils.cpython-312.pyc,,
+pip/_vendor/packaging/__pycache__/version.cpython-312.pyc,,
+pip/_vendor/packaging/_elffile.py,sha256=UkrbDtW7aeq3qqoAfU16ojyHZ1xsTvGke_WqMTKAKd0,3286
+pip/_vendor/packaging/_manylinux.py,sha256=t4y_-dTOcfr36gLY-ztiOpxxJFGO2ikC11HgfysGxiM,9596
+pip/_vendor/packaging/_musllinux.py,sha256=p9ZqNYiOItGee8KcZFeHF_YcdhVwGHdK6r-8lgixvGQ,2694
+pip/_vendor/packaging/_parser.py,sha256=gYfnj0pRHflVc4RHZit13KNTyN9iiVcU2RUCGi22BwM,10221
+pip/_vendor/packaging/_structures.py,sha256=q3eVNmbWJGG_S0Dit_S3Ao8qQqz_5PYTXFAKBZe5yr4,1431
+pip/_vendor/packaging/_tokenizer.py,sha256=OYzt7qKxylOAJ-q0XyK1qAycyPRYLfMPdGQKRXkZWyI,5310
+pip/_vendor/packaging/licenses/__init__.py,sha256=3bx-gryo4sRv5LsrwApouy65VIs3u6irSORJzALkrzU,5727
+pip/_vendor/packaging/licenses/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/packaging/licenses/__pycache__/_spdx.cpython-312.pyc,,
+pip/_vendor/packaging/licenses/_spdx.py,sha256=oAm1ztPFwlsmCKe7lAAsv_OIOfS1cWDu9bNBkeu-2ns,48398
+pip/_vendor/packaging/markers.py,sha256=P0we27jm1xUzgGMJxBjtUFCIWeBxTsMeJTOJ6chZmAY,12049
+pip/_vendor/packaging/metadata.py,sha256=8IZErqQQnNm53dZZuYq4FGU4_dpyinMeH1QFBIWIkfE,34739
+pip/_vendor/packaging/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/packaging/requirements.py,sha256=gYyRSAdbrIyKDY66ugIDUQjRMvxkH2ALioTmX3tnL6o,2947
+pip/_vendor/packaging/specifiers.py,sha256=yc9D_MycJEmwUpZvcs1OZL9HfiNFmyw0RZaeHRNHkPw,40079
+pip/_vendor/packaging/tags.py,sha256=41s97W9Zatrq2Ed7Rc3qeBDaHe8pKKvYq2mGjwahfXk,22745
+pip/_vendor/packaging/utils.py,sha256=0F3Hh9OFuRgrhTgGZUl5K22Fv1YP2tZl1z_2gO6kJiA,5050
+pip/_vendor/packaging/version.py,sha256=oiHqzTUv_p12hpjgsLDVcaF5hT7pDaSOViUNMD4GTW0,16688
+pip/_vendor/pkg_resources/__init__.py,sha256=vbTJ0_ruUgGxQjlEqsruFmiNPVyh2t9q-zyTDT053xI,124451
+pip/_vendor/pkg_resources/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/platformdirs/__init__.py,sha256=UfeSHWl8AeTtbOBOoHAxK4dODOWkZtfy-m_i7cWdJ8c,22344
+pip/_vendor/platformdirs/__main__.py,sha256=jBJ8zb7Mpx5ebcqF83xrpO94MaeCpNGHVf9cvDN2JLg,1505
+pip/_vendor/platformdirs/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/platformdirs/__pycache__/__main__.cpython-312.pyc,,
+pip/_vendor/platformdirs/__pycache__/android.cpython-312.pyc,,
+pip/_vendor/platformdirs/__pycache__/api.cpython-312.pyc,,
+pip/_vendor/platformdirs/__pycache__/macos.cpython-312.pyc,,
+pip/_vendor/platformdirs/__pycache__/unix.cpython-312.pyc,,
+pip/_vendor/platformdirs/__pycache__/version.cpython-312.pyc,,
+pip/_vendor/platformdirs/__pycache__/windows.cpython-312.pyc,,
+pip/_vendor/platformdirs/android.py,sha256=r0DshVBf-RO1jXJGX8C4Til7F1XWt-bkdWMgmvEiaYg,9013
+pip/_vendor/platformdirs/api.py,sha256=U9EzI3EYxcXWUCtIGRllqrcN99i2LSY1mq2-GtsUwEQ,9277
+pip/_vendor/platformdirs/macos.py,sha256=UlbyFZ8Rzu3xndCqQEHrfsYTeHwYdFap1Ioz-yxveT4,6154
+pip/_vendor/platformdirs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/platformdirs/unix.py,sha256=WZmkUA--L3JNRGmz32s35YfoD3ica6xKIPdCV_HhLcs,10458
+pip/_vendor/platformdirs/version.py,sha256=ddN3EcUPfer7CbqmyFNmg03R3u-qDn32T_fLsx25-Ck,511
+pip/_vendor/platformdirs/windows.py,sha256=IFpiohUBwxPtCzlyKwNtxyW4Jk8haa6W8o59mfrDXVo,10125
+pip/_vendor/pygments/__init__.py,sha256=8uNqJCCwXqbEx5aSsBr0FykUQOBDKBihO5mPqiw1aqo,2983
+pip/_vendor/pygments/__main__.py,sha256=WrndpSe6i1ckX_SQ1KaxD9CTKGzD0EuCOFxcbwFpoLU,353
+pip/_vendor/pygments/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/__main__.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/console.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/filter.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/formatter.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/lexer.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/modeline.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/plugin.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/regexopt.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/scanner.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/sphinxext.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/style.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/token.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/unistring.cpython-312.pyc,,
+pip/_vendor/pygments/__pycache__/util.cpython-312.pyc,,
+pip/_vendor/pygments/console.py,sha256=AagDWqwea2yBWf10KC9ptBgMpMjxKp8yABAmh-NQOVk,1718
+pip/_vendor/pygments/filter.py,sha256=YLtpTnZiu07nY3oK9nfR6E9Y1FBHhP5PX8gvkJWcfag,1910
+pip/_vendor/pygments/filters/__init__.py,sha256=4U4jtA0X3iP83uQnB9-TI-HDSw8E8y8zMYHa0UjbbaI,40392
+pip/_vendor/pygments/filters/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/pygments/formatter.py,sha256=KZQMmyo_xkOIkQG8g66LYEkBh1bx7a0HyGCBcvhI9Ew,4390
+pip/_vendor/pygments/formatters/__init__.py,sha256=KTwBmnXlaopJhQDOemVHYHskiDghuq-08YtP6xPNJPg,5385
+pip/_vendor/pygments/formatters/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/pygments/formatters/__pycache__/_mapping.cpython-312.pyc,,
+pip/_vendor/pygments/formatters/_mapping.py,sha256=1Cw37FuQlNacnxRKmtlPX4nyLoX9_ttko5ZwscNUZZ4,4176
+pip/_vendor/pygments/lexer.py,sha256=_kBrOJ_NT5Tl0IVM0rA9c8eysP6_yrlGzEQI0eVYB-A,35349
+pip/_vendor/pygments/lexers/__init__.py,sha256=wbIME35GH7bI1B9rNPJFqWT-ij_RApZDYPUlZycaLzA,12115
+pip/_vendor/pygments/lexers/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/pygments/lexers/__pycache__/_mapping.cpython-312.pyc,,
+pip/_vendor/pygments/lexers/__pycache__/python.cpython-312.pyc,,
+pip/_vendor/pygments/lexers/_mapping.py,sha256=l4tCXM8e9aPC2BD6sjIr0deT-J-z5tHgCwL-p1fS0PE,77602
+pip/_vendor/pygments/lexers/python.py,sha256=vxjn1cOHclIKJKxoyiBsQTY65GHbkZtZRuKQ2AVCKaw,53853
+pip/_vendor/pygments/modeline.py,sha256=K5eSkR8GS1r5OkXXTHOcV0aM_6xpk9eWNEIAW-OOJ2g,1005
+pip/_vendor/pygments/plugin.py,sha256=tPx0rJCTIZ9ioRgLNYG4pifCbAwTRUZddvLw-NfAk2w,1891
+pip/_vendor/pygments/regexopt.py,sha256=wXaP9Gjp_hKAdnICqoDkRxAOQJSc4v3X6mcxx3z-TNs,3072
+pip/_vendor/pygments/scanner.py,sha256=nNcETRR1tRuiTaHmHSTTECVYFPcLf6mDZu1e4u91A9E,3092
+pip/_vendor/pygments/sphinxext.py,sha256=5x7Zh9YlU6ISJ31dMwduiaanb5dWZnKg3MyEQsseNnQ,7981
+pip/_vendor/pygments/style.py,sha256=PlOZqlsnTVd58RGy50vkA2cXQ_lP5bF5EGMEBTno6DA,6420
+pip/_vendor/pygments/styles/__init__.py,sha256=x9ebctfyvCAFpMTlMJ5YxwcNYBzjgq6zJaKkNm78r4M,2042
+pip/_vendor/pygments/styles/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/pygments/styles/__pycache__/_mapping.cpython-312.pyc,,
+pip/_vendor/pygments/styles/_mapping.py,sha256=6lovFUE29tz6EsV3XYY4hgozJ7q1JL7cfO3UOlgnS8w,3312
+pip/_vendor/pygments/token.py,sha256=WbdWGhYm_Vosb0DDxW9lHNPgITXfWTsQmHt6cy9RbcM,6226
+pip/_vendor/pygments/unistring.py,sha256=al-_rBemRuGvinsrM6atNsHTmJ6DUbw24q2O2Ru1cBc,63208
+pip/_vendor/pygments/util.py,sha256=oRtSpiAo5jM9ulntkvVbgXUdiAW57jnuYGB7t9fYuhc,10031
+pip/_vendor/pyproject_hooks/__init__.py,sha256=cPB_a9LXz5xvsRbX1o2qyAdjLatZJdQ_Lc5McNX-X7Y,691
+pip/_vendor/pyproject_hooks/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/pyproject_hooks/__pycache__/_impl.cpython-312.pyc,,
+pip/_vendor/pyproject_hooks/_impl.py,sha256=jY-raxnmyRyB57ruAitrJRUzEexuAhGTpgMygqx67Z4,14936
+pip/_vendor/pyproject_hooks/_in_process/__init__.py,sha256=MJNPpfIxcO-FghxpBbxkG1rFiQf6HOUbV4U5mq0HFns,557
+pip/_vendor/pyproject_hooks/_in_process/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/pyproject_hooks/_in_process/__pycache__/_in_process.cpython-312.pyc,,
+pip/_vendor/pyproject_hooks/_in_process/_in_process.py,sha256=qcXMhmx__MIJq10gGHW3mA4Tl8dy8YzHMccwnNoKlw0,12216
+pip/_vendor/pyproject_hooks/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/requests/__init__.py,sha256=HlB_HzhrzGtfD_aaYUwUh1zWXLZ75_YCLyit75d0Vz8,5057
+pip/_vendor/requests/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/__version__.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/_internal_utils.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/adapters.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/api.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/auth.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/certs.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/compat.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/cookies.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/exceptions.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/help.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/hooks.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/models.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/packages.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/sessions.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/status_codes.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/structures.cpython-312.pyc,,
+pip/_vendor/requests/__pycache__/utils.cpython-312.pyc,,
+pip/_vendor/requests/__version__.py,sha256=FDq681Y3EvBjdDp5UqplMZ28uTTYlM_Jib0sAV-NpXc,435
+pip/_vendor/requests/_internal_utils.py,sha256=nMQymr4hs32TqVo5AbCrmcJEhvPUh7xXlluyqwslLiQ,1495
+pip/_vendor/requests/adapters.py,sha256=J7VeVxKBvawbtlX2DERVo05J9BXTcWYLMHNd1Baa-bk,27607
+pip/_vendor/requests/api.py,sha256=_Zb9Oa7tzVIizTKwFrPjDEY9ejtm_OnSRERnADxGsQs,6449
+pip/_vendor/requests/auth.py,sha256=kF75tqnLctZ9Mf_hm9TZIj4cQWnN5uxRz8oWsx5wmR0,10186
+pip/_vendor/requests/certs.py,sha256=kHDlkK_beuHXeMPc5jta2wgl8gdKeUWt5f2nTDVrvt8,441
+pip/_vendor/requests/compat.py,sha256=QfbmdTFiZzjSHMXiMrd4joCRU6RabtQ9zIcPoVaHIus,1822
+pip/_vendor/requests/cookies.py,sha256=bNi-iqEj4NPZ00-ob-rHvzkvObzN3lEpgw3g6paS3Xw,18590
+pip/_vendor/requests/exceptions.py,sha256=D1wqzYWne1mS2rU43tP9CeN1G7QAy7eqL9o1god6Ejw,4272
+pip/_vendor/requests/help.py,sha256=hRKaf9u0G7fdwrqMHtF3oG16RKktRf6KiwtSq2Fo1_0,3813
+pip/_vendor/requests/hooks.py,sha256=CiuysiHA39V5UfcCBXFIx83IrDpuwfN9RcTUgv28ftQ,733
+pip/_vendor/requests/models.py,sha256=taljlg6vJ4b-xMu2TaMNFFkaiwMex_VsEQ6qUTN3wzY,35575
+pip/_vendor/requests/packages.py,sha256=_ZQDCJTJ8SP3kVWunSqBsRZNPzj2c1WFVqbdr08pz3U,1057
+pip/_vendor/requests/sessions.py,sha256=ykTI8UWGSltOfH07HKollH7kTBGw4WhiBVaQGmckTw4,30495
+pip/_vendor/requests/status_codes.py,sha256=iJUAeA25baTdw-6PfD0eF4qhpINDJRJI-yaMqxs4LEI,4322
+pip/_vendor/requests/structures.py,sha256=-IbmhVz06S-5aPSZuUthZ6-6D9XOjRuTXHOabY041XM,2912
+pip/_vendor/requests/utils.py,sha256=WS3wHSQaaEfceu1syiFo5jf4e_CWKUTep_IabOVI_J0,33225
+pip/_vendor/resolvelib/__init__.py,sha256=T0LcAr9Sfai6ZXanpwavdHEea-Anw2QZGx16zd3lMKY,541
+pip/_vendor/resolvelib/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/resolvelib/__pycache__/providers.cpython-312.pyc,,
+pip/_vendor/resolvelib/__pycache__/reporters.cpython-312.pyc,,
+pip/_vendor/resolvelib/__pycache__/structs.cpython-312.pyc,,
+pip/_vendor/resolvelib/providers.py,sha256=pIWJbIdJJ9GFtNbtwTH0Ia43Vj6hYCEJj2DOLue15FM,8914
+pip/_vendor/resolvelib/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/resolvelib/reporters.py,sha256=pNJf4nFxLpAeKxlBUi2GEj0a2Ij1nikY0UabTKXesT4,2037
+pip/_vendor/resolvelib/resolvers/__init__.py,sha256=728M3EvmnPbVXS7ExXlv2kMu6b7wEsoPutEfl-uVk_I,640
+pip/_vendor/resolvelib/resolvers/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/resolvelib/resolvers/__pycache__/abstract.cpython-312.pyc,,
+pip/_vendor/resolvelib/resolvers/__pycache__/criterion.cpython-312.pyc,,
+pip/_vendor/resolvelib/resolvers/__pycache__/exceptions.cpython-312.pyc,,
+pip/_vendor/resolvelib/resolvers/__pycache__/resolution.cpython-312.pyc,,
+pip/_vendor/resolvelib/resolvers/abstract.py,sha256=jZOBVigE4PUub9i3F-bTvBwaIXX8S9EU3CGASBvFqEU,1558
+pip/_vendor/resolvelib/resolvers/criterion.py,sha256=lcmZGv5sKHOnFD_RzZwvlGSj19MeA-5rCMpdf2Sgw7Y,1768
+pip/_vendor/resolvelib/resolvers/exceptions.py,sha256=ln_jaQtgLlRUSFY627yiHG2gD7AgaXzRKaElFVh7fDQ,1768
+pip/_vendor/resolvelib/resolvers/resolution.py,sha256=qU64VKtN-HxoFynmnI6oP8aMPzaiKZtb_1hASH3HTHI,23994
+pip/_vendor/resolvelib/structs.py,sha256=pu-EJiR2IBITr2SQeNPRa0rXhjlStfmO_GEgAhr3004,6420
+pip/_vendor/rich/__init__.py,sha256=dRxjIL-SbFVY0q3IjSMrfgBTHrm1LZDgLOygVBwiYZc,6090
+pip/_vendor/rich/__main__.py,sha256=e_aVC-tDzarWQW9SuZMuCgBr6ODV_iDNV2Wh2xkxOlw,7896
+pip/_vendor/rich/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/__main__.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_cell_widths.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_emoji_codes.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_emoji_replace.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_export_format.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_extension.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_fileno.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_inspect.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_log_render.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_loop.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_null_file.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_palettes.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_pick.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_ratio.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_spinners.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_stack.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_timer.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_win32_console.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_windows.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_windows_renderer.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/_wrap.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/abc.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/align.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/ansi.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/bar.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/box.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/cells.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/color.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/color_triplet.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/columns.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/console.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/constrain.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/containers.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/control.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/default_styles.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/diagnose.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/emoji.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/errors.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/file_proxy.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/filesize.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/highlighter.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/json.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/jupyter.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/layout.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/live.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/live_render.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/logging.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/markup.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/measure.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/padding.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/pager.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/palette.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/panel.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/pretty.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/progress.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/progress_bar.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/prompt.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/protocol.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/region.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/repr.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/rule.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/scope.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/screen.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/segment.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/spinner.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/status.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/style.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/styled.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/syntax.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/table.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/terminal_theme.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/text.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/theme.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/themes.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/traceback.cpython-312.pyc,,
+pip/_vendor/rich/__pycache__/tree.cpython-312.pyc,,
+pip/_vendor/rich/_cell_widths.py,sha256=fbmeyetEdHjzE_Vx2l1uK7tnPOhMs2X1lJfO3vsKDpA,10209
+pip/_vendor/rich/_emoji_codes.py,sha256=hu1VL9nbVdppJrVoijVshRlcRRe_v3dju3Mmd2sKZdY,140235
+pip/_vendor/rich/_emoji_replace.py,sha256=n-kcetsEUx2ZUmhQrfeMNc-teeGhpuSQ5F8VPBsyvDo,1064
+pip/_vendor/rich/_export_format.py,sha256=RI08pSrm5tBSzPMvnbTqbD9WIalaOoN5d4M1RTmLq1Y,2128
+pip/_vendor/rich/_extension.py,sha256=Xt47QacCKwYruzjDi-gOBq724JReDj9Cm9xUi5fr-34,265
+pip/_vendor/rich/_fileno.py,sha256=HWZxP5C2ajMbHryvAQZseflVfQoGzsKOHzKGsLD8ynQ,799
+pip/_vendor/rich/_inspect.py,sha256=ROT0PLC2GMWialWZkqJIjmYq7INRijQQkoSokWTaAiI,9656
+pip/_vendor/rich/_log_render.py,sha256=1ByI0PA1ZpxZY3CGJOK54hjlq4X-Bz_boIjIqCd8Kns,3225
+pip/_vendor/rich/_loop.py,sha256=hV_6CLdoPm0va22Wpw4zKqM0RYsz3TZxXj0PoS-9eDQ,1236
+pip/_vendor/rich/_null_file.py,sha256=ADGKp1yt-k70FMKV6tnqCqecB-rSJzp-WQsD7LPL-kg,1394
+pip/_vendor/rich/_palettes.py,sha256=cdev1JQKZ0JvlguV9ipHgznTdnvlIzUFDBb0It2PzjI,7063
+pip/_vendor/rich/_pick.py,sha256=evDt8QN4lF5CiwrUIXlOJCntitBCOsI3ZLPEIAVRLJU,423
+pip/_vendor/rich/_ratio.py,sha256=IOtl78sQCYZsmHyxhe45krkb68u9xVz7zFsXVJD-b2Y,5325
+pip/_vendor/rich/_spinners.py,sha256=U2r1_g_1zSjsjiUdAESc2iAMc3i4ri_S8PYP6kQ5z1I,19919
+pip/_vendor/rich/_stack.py,sha256=-C8OK7rxn3sIUdVwxZBBpeHhIzX0eI-VM3MemYfaXm0,351
+pip/_vendor/rich/_timer.py,sha256=zelxbT6oPFZnNrwWPpc1ktUeAT-Vc4fuFcRZLQGLtMI,417
+pip/_vendor/rich/_win32_console.py,sha256=BSaDRIMwBLITn_m0mTRLPqME5q-quGdSMuYMpYeYJwc,22755
+pip/_vendor/rich/_windows.py,sha256=aBwaD_S56SbgopIvayVmpk0Y28uwY2C5Bab1wl3Bp-I,1925
+pip/_vendor/rich/_windows_renderer.py,sha256=t74ZL3xuDCP3nmTp9pH1L5LiI2cakJuQRQleHCJerlk,2783
+pip/_vendor/rich/_wrap.py,sha256=FlSsom5EX0LVkA3KWy34yHnCfLtqX-ZIepXKh-70rpc,3404
+pip/_vendor/rich/abc.py,sha256=ON-E-ZqSSheZ88VrKX2M3PXpFbGEUUZPMa_Af0l-4f0,890
+pip/_vendor/rich/align.py,sha256=dg-7uY0ukMLLlUEsBDRLva22_sQgIJD4BK0dmZHFHug,10324
+pip/_vendor/rich/ansi.py,sha256=Avs1LHbSdcyOvDOdpELZUoULcBiYewY76eNBp6uFBhs,6921
+pip/_vendor/rich/bar.py,sha256=ldbVHOzKJOnflVNuv1xS7g6dLX2E3wMnXkdPbpzJTcs,3263
+pip/_vendor/rich/box.py,sha256=kmavBc_dn73L_g_8vxWSwYJD2uzBXOUFTtJOfpbczcM,10686
+pip/_vendor/rich/cells.py,sha256=KrQkj5-LghCCpJLSNQIyAZjndc4bnEqOEmi5YuZ9UCY,5130
+pip/_vendor/rich/color.py,sha256=3HSULVDj7qQkXUdFWv78JOiSZzfy5y1nkcYhna296V0,18211
+pip/_vendor/rich/color_triplet.py,sha256=3lhQkdJbvWPoLDO-AnYImAWmJvV5dlgYNCVZ97ORaN4,1054
+pip/_vendor/rich/columns.py,sha256=HUX0KcMm9dsKNi11fTbiM_h2iDtl8ySCaVcxlalEzq8,7131
+pip/_vendor/rich/console.py,sha256=t9azZpmRMVU5cphVBZSShNsmBxd2-IAWcTTlhor-E1s,100849
+pip/_vendor/rich/constrain.py,sha256=1VIPuC8AgtKWrcncQrjBdYqA3JVWysu6jZo1rrh7c7Q,1288
+pip/_vendor/rich/containers.py,sha256=c_56TxcedGYqDepHBMTuZdUIijitAQgnox-Qde0Z1qo,5502
+pip/_vendor/rich/control.py,sha256=EUTSUFLQbxY6Zmo_sdM-5Ls323vIHTBfN8TPulqeHUY,6487
+pip/_vendor/rich/default_styles.py,sha256=khQFqqaoDs3bprMqWpHw8nO5UpG2DN6QtuTd6LzZwYc,8257
+pip/_vendor/rich/diagnose.py,sha256=fJl1TItRn19gGwouqTg-8zPUW3YqQBqGltrfPQs1H9w,1025
+pip/_vendor/rich/emoji.py,sha256=Wd4bQubZdSy6-PyrRQNuMHtn2VkljK9uPZPVlu2cmx0,2367
+pip/_vendor/rich/errors.py,sha256=5pP3Kc5d4QJ_c0KFsxrfyhjiPVe7J1zOqSFbFAzcV-Y,642
+pip/_vendor/rich/file_proxy.py,sha256=Tl9THMDZ-Pk5Wm8sI1gGg_U5DhusmxD-FZ0fUbcU0W0,1683
+pip/_vendor/rich/filesize.py,sha256=_iz9lIpRgvW7MNSeCZnLg-HwzbP4GETg543WqD8SFs0,2484
+pip/_vendor/rich/highlighter.py,sha256=G_sn-8DKjM1sEjLG_oc4ovkWmiUpWvj8bXi0yed2LnY,9586
+pip/_vendor/rich/json.py,sha256=vVEoKdawoJRjAFayPwXkMBPLy7RSTs-f44wSQDR2nJ0,5031
+pip/_vendor/rich/jupyter.py,sha256=QyoKoE_8IdCbrtiSHp9TsTSNyTHY0FO5whE7jOTd9UE,3252
+pip/_vendor/rich/layout.py,sha256=ajkSFAtEVv9EFTcFs-w4uZfft7nEXhNzL7ZVdgrT5rI,14004
+pip/_vendor/rich/live.py,sha256=tF3ukAAJZ_N2ZbGclqZ-iwLoIoZ8f0HHUz79jAyJqj8,15180
+pip/_vendor/rich/live_render.py,sha256=It_39YdzrBm8o3LL0kaGorPFg-BfZWAcrBjLjFokbx4,3521
+pip/_vendor/rich/logging.py,sha256=5KaPPSMP9FxcXPBcKM4cGd_zW78PMgf-YbMVnvfSw0o,12468
+pip/_vendor/rich/markup.py,sha256=3euGKP5s41NCQwaSjTnJxus5iZMHjxpIM0W6fCxra38,8451
+pip/_vendor/rich/measure.py,sha256=HmrIJX8sWRTHbgh8MxEay_83VkqNW_70s8aKP5ZcYI8,5305
+pip/_vendor/rich/padding.py,sha256=KVEI3tOwo9sgK1YNSuH__M1_jUWmLZwRVV_KmOtVzyM,4908
+pip/_vendor/rich/pager.py,sha256=SO_ETBFKbg3n_AgOzXm41Sv36YxXAyI3_R-KOY2_uSc,828
+pip/_vendor/rich/palette.py,sha256=lInvR1ODDT2f3UZMfL1grq7dY_pDdKHw4bdUgOGaM4Y,3396
+pip/_vendor/rich/panel.py,sha256=9sQl00hPIqH5G2gALQo4NepFwpP0k9wT-s_gOms5pIc,11157
+pip/_vendor/rich/pretty.py,sha256=gy3S72u4FRg2ytoo7N1ZDWDIvB4unbzd5iUGdgm-8fc,36391
+pip/_vendor/rich/progress.py,sha256=CUc2lkU-X59mVdGfjMCBkZeiGPL3uxdONjhNJF2T7wY,60408
+pip/_vendor/rich/progress_bar.py,sha256=mZTPpJUwcfcdgQCTTz3kyY-fc79ddLwtx6Ghhxfo064,8162
+pip/_vendor/rich/prompt.py,sha256=l0RhQU-0UVTV9e08xW1BbIj0Jq2IXyChX4lC0lFNzt4,12447
+pip/_vendor/rich/protocol.py,sha256=5hHHDDNHckdk8iWH5zEbi-zuIVSF5hbU2jIo47R7lTE,1391
+pip/_vendor/rich/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/rich/region.py,sha256=rNT9xZrVZTYIXZC0NYn41CJQwYNbR-KecPOxTgQvB8Y,166
+pip/_vendor/rich/repr.py,sha256=5MZJZmONgC6kud-QW-_m1okXwL2aR6u6y-pUcUCJz28,4431
+pip/_vendor/rich/rule.py,sha256=0fNaS_aERa3UMRc3T5WMpN_sumtDxfaor2y3of1ftBk,4602
+pip/_vendor/rich/scope.py,sha256=TMUU8qo17thyqQCPqjDLYpg_UU1k5qVd-WwiJvnJVas,2843
+pip/_vendor/rich/screen.py,sha256=YoeReESUhx74grqb0mSSb9lghhysWmFHYhsbMVQjXO8,1591
+pip/_vendor/rich/segment.py,sha256=otnKeKGEV-WRlQVosfJVeFDcDxAKHpvJ_hLzSu5lumM,24743
+pip/_vendor/rich/spinner.py,sha256=onIhpKlljRHppTZasxO8kXgtYyCHUkpSgKglRJ3o51g,4214
+pip/_vendor/rich/status.py,sha256=kkPph3YeAZBo-X-4wPp8gTqZyU466NLwZBA4PZTTewo,4424
+pip/_vendor/rich/style.py,sha256=xpj4uMBZMtuNuNomfUiamigl3p1sDvTCZwrG1tcTVeg,27059
+pip/_vendor/rich/styled.py,sha256=eZNnzGrI4ki_54pgY3Oj0T-x3lxdXTYh4_ryDB24wBU,1258
+pip/_vendor/rich/syntax.py,sha256=eDKIRwl--eZ0Lwo2da2RRtfutXGavrJO61Cl5OkS59U,36371
+pip/_vendor/rich/table.py,sha256=ZmT7V7MMCOYKw7TGY9SZLyYDf6JdM-WVf07FdVuVhTI,40049
+pip/_vendor/rich/terminal_theme.py,sha256=1j5-ufJfnvlAo5Qsi_ACZiXDmwMXzqgmFByObT9-yJY,3370
+pip/_vendor/rich/text.py,sha256=AO7JPCz6-gaN1thVLXMBntEmDPVYFgFNG1oM61_sanU,47552
+pip/_vendor/rich/theme.py,sha256=oNyhXhGagtDlbDye3tVu3esWOWk0vNkuxFw-_unlaK0,3771
+pip/_vendor/rich/themes.py,sha256=0xgTLozfabebYtcJtDdC5QkX5IVUEaviqDUJJh4YVFk,102
+pip/_vendor/rich/traceback.py,sha256=c0WmB_L04_UfZbLaoH982_U_s7eosxKMUiAVmDPdRYU,35861
+pip/_vendor/rich/tree.py,sha256=yWnQ6rAvRGJ3qZGqBrxS2SW2TKBTNrP0SdY8QxOFPuw,9451
+pip/_vendor/tomli/__init__.py,sha256=PhNw_eyLgdn7McJ6nrAN8yIm3dXC75vr1sVGVVwDSpA,314
+pip/_vendor/tomli/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/tomli/__pycache__/_parser.cpython-312.pyc,,
+pip/_vendor/tomli/__pycache__/_re.cpython-312.pyc,,
+pip/_vendor/tomli/__pycache__/_types.cpython-312.pyc,,
+pip/_vendor/tomli/_parser.py,sha256=9w8LG0jB7fwmZZWB0vVXbeejDHcl4ANIJxB2scEnDlA,25591
+pip/_vendor/tomli/_re.py,sha256=sh4sBDRgO94KJZwNIrgdcyV_qQast50YvzOAUGpRDKA,3171
+pip/_vendor/tomli/_types.py,sha256=-GTG2VUqkpxwMqzmVO4F7ybKddIbAnuAHXfmWQcTi3Q,254
+pip/_vendor/tomli/py.typed,sha256=8PjyZ1aVoQpRVvt71muvuq5qE-jTFZkK-GLHkhdebmc,26
+pip/_vendor/tomli_w/__init__.py,sha256=0F8yDtXx3Uunhm874KrAcP76srsM98y7WyHQwCulZbo,169
+pip/_vendor/tomli_w/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/tomli_w/__pycache__/_writer.cpython-312.pyc,,
+pip/_vendor/tomli_w/_writer.py,sha256=dsifFS2xYf1i76mmRyfz9y125xC7Z_HQ845ZKhJsYXs,6961
+pip/_vendor/tomli_w/py.typed,sha256=8PjyZ1aVoQpRVvt71muvuq5qE-jTFZkK-GLHkhdebmc,26
+pip/_vendor/truststore/__init__.py,sha256=2wRSVijjRzPLVXUzWqvdZLNsEOhDfopKLd2EKAYLwKU,1320
+pip/_vendor/truststore/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/truststore/__pycache__/_api.cpython-312.pyc,,
+pip/_vendor/truststore/__pycache__/_macos.cpython-312.pyc,,
+pip/_vendor/truststore/__pycache__/_openssl.cpython-312.pyc,,
+pip/_vendor/truststore/__pycache__/_ssl_constants.cpython-312.pyc,,
+pip/_vendor/truststore/__pycache__/_windows.cpython-312.pyc,,
+pip/_vendor/truststore/_api.py,sha256=af8gEZG_vhsudia9vz4es3Vh8xAqhzQz4Cbjs6_rxus,11234
+pip/_vendor/truststore/_macos.py,sha256=nZlLkOmszUE0g6ryRwBVGY5COzPyudcsiJtDWarM5LQ,20503
+pip/_vendor/truststore/_openssl.py,sha256=LLUZ7ZGaio-i5dpKKjKCSeSufmn6T8pi9lDcFnvSyq0,2324
+pip/_vendor/truststore/_ssl_constants.py,sha256=NUD4fVKdSD02ri7-db0tnO0VqLP9aHuzmStcW7tAl08,1130
+pip/_vendor/truststore/_windows.py,sha256=rAHyKYD8M7t-bXfG8VgOVa3TpfhVhbt4rZQlO45YuP8,17993
+pip/_vendor/truststore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/urllib3/__init__.py,sha256=iXLcYiJySn0GNbWOOZDDApgBL1JgP44EZ8i1760S8Mc,3333
+pip/_vendor/urllib3/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/urllib3/__pycache__/_collections.cpython-312.pyc,,
+pip/_vendor/urllib3/__pycache__/_version.cpython-312.pyc,,
+pip/_vendor/urllib3/__pycache__/connection.cpython-312.pyc,,
+pip/_vendor/urllib3/__pycache__/connectionpool.cpython-312.pyc,,
+pip/_vendor/urllib3/__pycache__/exceptions.cpython-312.pyc,,
+pip/_vendor/urllib3/__pycache__/fields.cpython-312.pyc,,
+pip/_vendor/urllib3/__pycache__/filepost.cpython-312.pyc,,
+pip/_vendor/urllib3/__pycache__/poolmanager.cpython-312.pyc,,
+pip/_vendor/urllib3/__pycache__/request.cpython-312.pyc,,
+pip/_vendor/urllib3/__pycache__/response.cpython-312.pyc,,
+pip/_vendor/urllib3/_collections.py,sha256=pyASJJhW7wdOpqJj9QJA8FyGRfr8E8uUUhqUvhF0728,11372
+pip/_vendor/urllib3/_version.py,sha256=t9wGB6ooOTXXgiY66K1m6BZS1CJyXHAU8EoWDTe6Shk,64
+pip/_vendor/urllib3/connection.py,sha256=ttIA909BrbTUzwkqEe_TzZVh4JOOj7g61Ysei2mrwGg,20314
+pip/_vendor/urllib3/connectionpool.py,sha256=e2eiAwNbFNCKxj4bwDKNK-w7HIdSz3OmMxU_TIt-evQ,40408
+pip/_vendor/urllib3/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/urllib3/contrib/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/urllib3/contrib/__pycache__/_appengine_environ.cpython-312.pyc,,
+pip/_vendor/urllib3/contrib/__pycache__/appengine.cpython-312.pyc,,
+pip/_vendor/urllib3/contrib/__pycache__/ntlmpool.cpython-312.pyc,,
+pip/_vendor/urllib3/contrib/__pycache__/pyopenssl.cpython-312.pyc,,
+pip/_vendor/urllib3/contrib/__pycache__/securetransport.cpython-312.pyc,,
+pip/_vendor/urllib3/contrib/__pycache__/socks.cpython-312.pyc,,
+pip/_vendor/urllib3/contrib/_appengine_environ.py,sha256=bDbyOEhW2CKLJcQqAKAyrEHN-aklsyHFKq6vF8ZFsmk,957
+pip/_vendor/urllib3/contrib/_securetransport/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/urllib3/contrib/_securetransport/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/urllib3/contrib/_securetransport/__pycache__/bindings.cpython-312.pyc,,
+pip/_vendor/urllib3/contrib/_securetransport/__pycache__/low_level.cpython-312.pyc,,
+pip/_vendor/urllib3/contrib/_securetransport/bindings.py,sha256=4Xk64qIkPBt09A5q-RIFUuDhNc9mXilVapm7WnYnzRw,17632
+pip/_vendor/urllib3/contrib/_securetransport/low_level.py,sha256=B2JBB2_NRP02xK6DCa1Pa9IuxrPwxzDzZbixQkb7U9M,13922
+pip/_vendor/urllib3/contrib/appengine.py,sha256=VR68eAVE137lxTgjBDwCna5UiBZTOKa01Aj_-5BaCz4,11036
+pip/_vendor/urllib3/contrib/ntlmpool.py,sha256=NlfkW7WMdW8ziqudopjHoW299og1BTWi0IeIibquFwk,4528
+pip/_vendor/urllib3/contrib/pyopenssl.py,sha256=hDJh4MhyY_p-oKlFcYcQaVQRDv6GMmBGuW9yjxyeejM,17081
+pip/_vendor/urllib3/contrib/securetransport.py,sha256=Fef1IIUUFHqpevzXiDPbIGkDKchY2FVKeVeLGR1Qq3g,34446
+pip/_vendor/urllib3/contrib/socks.py,sha256=aRi9eWXo9ZEb95XUxef4Z21CFlnnjbEiAo9HOseoMt4,7097
+pip/_vendor/urllib3/exceptions.py,sha256=0Mnno3KHTNfXRfY7638NufOPkUb6mXOm-Lqj-4x2w8A,8217
+pip/_vendor/urllib3/fields.py,sha256=kvLDCg_JmH1lLjUUEY_FLS8UhY7hBvDPuVETbY8mdrM,8579
+pip/_vendor/urllib3/filepost.py,sha256=5b_qqgRHVlL7uLtdAYBzBh-GHmU5AfJVt_2N0XS3PeY,2440
+pip/_vendor/urllib3/packages/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/urllib3/packages/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/urllib3/packages/__pycache__/six.cpython-312.pyc,,
+pip/_vendor/urllib3/packages/backports/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pip/_vendor/urllib3/packages/backports/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/urllib3/packages/backports/__pycache__/makefile.cpython-312.pyc,,
+pip/_vendor/urllib3/packages/backports/__pycache__/weakref_finalize.cpython-312.pyc,,
+pip/_vendor/urllib3/packages/backports/makefile.py,sha256=nbzt3i0agPVP07jqqgjhaYjMmuAi_W5E0EywZivVO8E,1417
+pip/_vendor/urllib3/packages/backports/weakref_finalize.py,sha256=tRCal5OAhNSRyb0DhHp-38AtIlCsRP8BxF3NX-6rqIA,5343
+pip/_vendor/urllib3/packages/six.py,sha256=b9LM0wBXv7E7SrbCjAm4wwN-hrH-iNxv18LgWNMMKPo,34665
+pip/_vendor/urllib3/poolmanager.py,sha256=aWyhXRtNO4JUnCSVVqKTKQd8EXTvUm1VN9pgs2bcONo,19990
+pip/_vendor/urllib3/request.py,sha256=YTWFNr7QIwh7E1W9dde9LM77v2VWTJ5V78XuTTw7D1A,6691
+pip/_vendor/urllib3/response.py,sha256=fmDJAFkG71uFTn-sVSTh2Iw0WmcXQYqkbRjihvwBjU8,30641
+pip/_vendor/urllib3/util/__init__.py,sha256=JEmSmmqqLyaw8P51gUImZh8Gwg9i1zSe-DoqAitn2nc,1155
+pip/_vendor/urllib3/util/__pycache__/__init__.cpython-312.pyc,,
+pip/_vendor/urllib3/util/__pycache__/connection.cpython-312.pyc,,
+pip/_vendor/urllib3/util/__pycache__/proxy.cpython-312.pyc,,
+pip/_vendor/urllib3/util/__pycache__/queue.cpython-312.pyc,,
+pip/_vendor/urllib3/util/__pycache__/request.cpython-312.pyc,,
+pip/_vendor/urllib3/util/__pycache__/response.cpython-312.pyc,,
+pip/_vendor/urllib3/util/__pycache__/retry.cpython-312.pyc,,
+pip/_vendor/urllib3/util/__pycache__/ssl_.cpython-312.pyc,,
+pip/_vendor/urllib3/util/__pycache__/ssl_match_hostname.cpython-312.pyc,,
+pip/_vendor/urllib3/util/__pycache__/ssltransport.cpython-312.pyc,,
+pip/_vendor/urllib3/util/__pycache__/timeout.cpython-312.pyc,,
+pip/_vendor/urllib3/util/__pycache__/url.cpython-312.pyc,,
+pip/_vendor/urllib3/util/__pycache__/wait.cpython-312.pyc,,
+pip/_vendor/urllib3/util/connection.py,sha256=5Lx2B1PW29KxBn2T0xkN1CBgRBa3gGVJBKoQoRogEVk,4901
+pip/_vendor/urllib3/util/proxy.py,sha256=zUvPPCJrp6dOF0N4GAVbOcl6o-4uXKSrGiTkkr5vUS4,1605
+pip/_vendor/urllib3/util/queue.py,sha256=nRgX8_eX-_VkvxoX096QWoz8Ps0QHUAExILCY_7PncM,498
+pip/_vendor/urllib3/util/request.py,sha256=C0OUt2tcU6LRiQJ7YYNP9GvPrSvl7ziIBekQ-5nlBZk,3997
+pip/_vendor/urllib3/util/response.py,sha256=GJpg3Egi9qaJXRwBh5wv-MNuRWan5BIu40oReoxWP28,3510
+pip/_vendor/urllib3/util/retry.py,sha256=6ENvOZ8PBDzh8kgixpql9lIrb2dxH-k7ZmBanJF2Ng4,22050
+pip/_vendor/urllib3/util/ssl_.py,sha256=QDuuTxPSCj1rYtZ4xpD7Ux-r20TD50aHyqKyhQ7Bq4A,17460
+pip/_vendor/urllib3/util/ssl_match_hostname.py,sha256=Ir4cZVEjmAk8gUAIHWSi7wtOO83UCYABY2xFD1Ql_WA,5758
+pip/_vendor/urllib3/util/ssltransport.py,sha256=NA-u5rMTrDFDFC8QzRKUEKMG0561hOD4qBTr3Z4pv6E,6895
+pip/_vendor/urllib3/util/timeout.py,sha256=cwq4dMk87mJHSBktK1miYJ-85G-3T3RmT20v7SFCpno,10168
+pip/_vendor/urllib3/util/url.py,sha256=lCAE7M5myA8EDdW0sJuyyZhVB9K_j38ljWhHAnFaWoE,14296
+pip/_vendor/urllib3/util/wait.py,sha256=fOX0_faozG2P7iVojQoE1mbydweNyTcm-hXEfFrTtLI,5403
+pip/_vendor/vendor.txt,sha256=fawq8T1XFfBhs4rjjSl4fUA3Px9P2mtG2evqqPyhbhc,343
+pip/py.typed,sha256=EBVvvPRTn_eIpz5e5QztSCdrMX7Qwd7VP93RSoIlZ2I,286
diff --git a/lib/python3.1/site-packages/pip-25.2.dist-info/REQUESTED b/lib/python3.1/site-packages/pip-25.2.dist-info/REQUESTED
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lib/python3.1/site-packages/pip-25.2.dist-info/WHEEL b/lib/python3.1/site-packages/pip-25.2.dist-info/WHEEL
new file mode 100644
index 0000000000000000000000000000000000000000..e7fa31b6f3f78deb1022c1f7927f07d4d16da822
--- /dev/null
+++ b/lib/python3.1/site-packages/pip-25.2.dist-info/WHEEL
@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+
diff --git a/lib/python3.1/site-packages/pip-25.2.dist-info/direct_url.json b/lib/python3.1/site-packages/pip-25.2.dist-info/direct_url.json
new file mode 100644
index 0000000000000000000000000000000000000000..55267362d856578e7c2668216bd4385db7526e70
--- /dev/null
+++ b/lib/python3.1/site-packages/pip-25.2.dist-info/direct_url.json
@@ -0,0 +1 @@
+{"dir_info": {}, "url": "file:///home/conda/feedstock_root/build_artifacts/pip_1753924886980/work"}
\ No newline at end of file
diff --git a/lib/python3.1/site-packages/pip-25.2.dist-info/entry_points.txt b/lib/python3.1/site-packages/pip-25.2.dist-info/entry_points.txt
new file mode 100644
index 0000000000000000000000000000000000000000..25fcf7e2cdf0943ad5c074d4777c721cd3340986
--- /dev/null
+++ b/lib/python3.1/site-packages/pip-25.2.dist-info/entry_points.txt
@@ -0,0 +1,3 @@
+[console_scripts]
+pip = pip._internal.cli.main:main
+pip3 = pip._internal.cli.main:main
diff --git a/lib/python3.1/site-packages/pip-25.2.dist-info/top_level.txt b/lib/python3.1/site-packages/pip-25.2.dist-info/top_level.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68
--- /dev/null
+++ b/lib/python3.1/site-packages/pip-25.2.dist-info/top_level.txt
@@ -0,0 +1 @@
+pip
diff --git a/lib/python3.1/site-packages/pycountry/COPYRIGHT.txt b/lib/python3.1/site-packages/pycountry/COPYRIGHT.txt
new file mode 100644
index 0000000000000000000000000000000000000000..7b6d508a7b4ff945cc285d445d693592c00c240d
--- /dev/null
+++ b/lib/python3.1/site-packages/pycountry/COPYRIGHT.txt
@@ -0,0 +1,46 @@
+COPYRIGHT (c) 2008 - 2023, pycountry
+
+Pycountry is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or any later version.
+
+This project is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Lesser General Public License for more details.
+
+Contributors:
+- Christian Theune (2008-2020, 2022)
+- Nate Schimmoller (2022-2023)
+- Zachary Ware (2016, 2023)
+- Alan Orth (2023)
+- Ashok Argent-Katwala (2020)
+- Bastien Vallet (2020)
+- Chris R Bunney 2020
+- Christian Zagrodnick (2012-2013)
+- Christoph Zwerschke (2013)
+- Jakub Wilk (2020)
+- Janis Kirsteins (2019)
+- Justin Ryan Wagner 2014
+- Kevin Deldycke (2014, 2016)
+- Louis Sautier (2020)
+- Lucas Wiman (2015)
+- Mario Vilas (2014)
+- Michael Howitz (2020)
+- Michał Bielawski (2021, 2023)
+- Michał Górny (2020)
+- Mike Taves (2023)
+- Pedro Ferreira (2013)
+- Stuart Prescott (2021)
+- Victor Mireyev (2016)
+- simon klemenc (2016)
+
+Additional Acknowledgements and Licensing Information:
+- Data in the /src/databases/ and /src/locales/ folders is sourced from the Debian iso-codes project, available at: https://salsa.debian.org/iso-codes-team/iso-codes. This data is used under the terms of the GNU Lesser General Public License Version 2.1 (February 1999).
+
+The Debian iso-codes project is a collection of code lists for different standards, maintained and made available under the GNU Lesser General Public License Version 2.1. We gratefully acknowledge the Debian iso-codes team and contributors for their work and for making this resource freely available.
+
+The full text of the GNU Lesser General Public License Version 2.1 can be found at: https://salsa.debian.org/iso-codes-team/iso-codes/-/blob/main/COPYING.
+
+For the full text of the GNU Lesser General Public License,
+see https://github.com/pycountry/pycountry/blob/main/LICENSE.txt.
diff --git a/lib/python3.1/site-packages/pycountry/__init__.py b/lib/python3.1/site-packages/pycountry/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..33c3b1d52a787b76901c9a75a255f7ed718229c6
--- /dev/null
+++ b/lib/python3.1/site-packages/pycountry/__init__.py
@@ -0,0 +1,303 @@
+"""pycountry"""
+
+import os.path
+import unicodedata
+from importlib import metadata as _importlib_metadata
+from typing import Dict, List, Optional, Type
+
+import pycountry.db
+
+# We prioritise importing the backported `importlib_resources`
+# because the function we use (`importlib.resources.files`) is only
+# available from Python 3.9, but the module itself exists since 3.7.
+# We install `importlib_resources` on Python < 3.9.
+# TODO: Remove usage of importlib_resources once support for Python 3.8 is dropped
+try:
+    import importlib_resources  # type: ignore
+except ModuleNotFoundError:
+    from importlib import resources as importlib_resources  # type: ignore
+
+
+def resource_filename(package_or_requirement: str, resource_name: str) -> str:
+    return str(
+        importlib_resources.files(package_or_requirement) / resource_name
+    )
+
+
+def get_version(distribution_name: str) -> Optional[str]:
+    try:
+        return _importlib_metadata.version(distribution_name)
+    except _importlib_metadata.PackageNotFoundError:
+        return "n/a"
+
+
+# Variable annotations
+LOCALES_DIR: str = resource_filename("pycountry", "locales")
+DATABASE_DIR: str = resource_filename("pycountry", "databases")
+__version__: Optional[str] = get_version("pycountry")
+
+
+def remove_accents(input_str: str) -> str:
+    output_str = input_str
+    if not input_str.isascii():
+        # Borrowed from https://stackoverflow.com/a/517974/1509718
+        nfkd_form = unicodedata.normalize("NFKD", input_str)
+        output_str = "".join(
+            [c for c in nfkd_form if not unicodedata.combining(c)]
+        )
+    return output_str
+
+
+class ExistingCountries(pycountry.db.Database):
+    """Provides access to an ISO 3166 database (Countries)."""
+
+    data_class = pycountry.db.Country
+    root_key = "3166-1"
+
+    def search_fuzzy(self, query: str) -> List[Type["ExistingCountries"]]:
+        query = remove_accents(query.strip().lower())
+
+        # A country-code to points mapping for later sorting countries
+        # based on the query's matching incidence.
+        results: dict[str, int] = {}
+
+        def add_result(country: "pycountry.db.Country", points: int) -> None:
+            results.setdefault(country.alpha_2, 0)
+            results[country.alpha_2] += points
+
+        # Prio 1: exact matches on country names
+        try:
+            add_result(self.lookup(query), 50)
+        except LookupError:
+            pass
+
+        # Prio 2: exact matches on subdivision names
+        match_subdivions = pycountry.Subdivisions.match(
+            self=subdivisions, query=query
+        )
+        for candidate in match_subdivions:
+            add_result(candidate.country, 49)
+
+        # Prio 3: partial matches on country names
+        for candidate in self:
+            # Higher priority for a match on the common name
+            for v in [
+                candidate._fields.get("name"),
+                candidate._fields.get("official_name"),
+                candidate._fields.get("comment"),
+            ]:
+                if v is not None:
+                    v = remove_accents(v.lower())
+                    if query in v:
+                        # This prefers countries with a match early in their name
+                        # and also balances against countries with a number of
+                        # partial matches and their name containing 'new' in the
+                        # middle
+                        add_result(
+                            candidate, max([5, 30 - (2 * v.find(query))])
+                        )
+                        break
+
+        # Prio 4: partial matches on subdivision names
+        partial_match_subdivisions = pycountry.Subdivisions.partial_match(
+            self=subdivisions, query=query
+        )
+        for candidate in partial_match_subdivisions:
+            v = candidate._fields.get("name")
+            v = remove_accents(v.lower())
+            if query in v:
+                add_result(candidate.country, max([1, 5 - v.find(query)]))
+
+        if not results:
+            raise LookupError(query)
+
+        sorted_results = [
+            self.get(alpha_2=x[0])
+            # sort by points first, by alpha2 code second, and to ensure stable
+            # results the negative value allows us to sort reversely on the
+            # points but ascending on the country code.
+            for x in sorted(results.items(), key=lambda x: (-x[1], x[0]))
+        ]
+        return sorted_results
+
+
+class HistoricCountries(ExistingCountries):
+    """Provides access to an ISO 3166-3 database
+    (Countries that have been removed from the standard)."""
+
+    data_class = pycountry.db.Country
+    root_key = "3166-3"
+
+
+class Scripts(pycountry.db.Database):
+    """Provides access to an ISO 15924 database (Scripts)."""
+
+    data_class = "Script"
+    root_key = "15924"
+
+
+class Currencies(pycountry.db.Database):
+    """Provides access to an ISO 4217 database (Currencies)."""
+
+    data_class = "Currency"
+    root_key = "4217"
+
+
+class Languages(pycountry.db.Database):
+    """Provides access to an ISO 639-1/2T/3 database (Languages)."""
+
+    no_index = ["status", "scope", "type", "inverted_name", "common_name"]
+
+    data_class = "Language"
+    root_key = "639-3"
+
+
+class LanguageFamilies(pycountry.db.Database):
+    """Provides access to an ISO 639-5 database
+    (Language Families and Groups)."""
+
+    data_class = "LanguageFamily"
+    root_key = "639-5"
+
+
+class SubdivisionHierarchy(pycountry.db.Data):
+    def __init__(self, **kw):
+        if "parent" in kw:
+            kw["parent_code"] = kw["parent"]
+        else:
+            kw["parent_code"] = None
+        super().__init__(**kw)
+        self.country_code = self.code.split("-")[0]
+        if self.parent_code is not None:
+            # Split the parent_code to check if the country_code is already present
+            parts = self.parent_code.split("-")
+            if parts[0] != self.country_code:
+                self.parent_code = f"{self.country_code}-{self.parent_code}"
+
+    @property
+    def country(self):
+        return countries.get(alpha_2=self.country_code)
+
+    @property
+    def parent(self):
+        if not self.parent_code:
+            return None
+        return subdivisions.get(code=self.parent_code)
+
+
+class Subdivisions(pycountry.db.Database):
+    # Note: subdivisions can be hierarchical to other subdivisions. The
+    # parent_code attribute is related to other subdivisions, *not*
+    # the country!
+
+    data_class = SubdivisionHierarchy
+    no_index = ["name", "parent_code", "parent", "type"]
+    root_key = "3166-2"
+
+    def _load(self, *args, **kw):
+        super()._load(*args, **kw)
+
+        # Add index for the country code.
+        self.indices["country_code"] = {}
+        for subdivision in self:
+            divs = self.indices["country_code"].setdefault(
+                subdivision.country_code.lower(), set()
+            )
+            divs.add(subdivision)
+
+    def get(self, **kw):
+        default = kw.setdefault("default", None)
+        subdivisions = super().get(**kw)
+        if subdivisions is default and "country_code" in kw:
+            # This handles the case where we know about a country but there
+            # are no subdivisions: we return an empty list in this case
+            # (sticking to the expected type here) instead of None.
+            if countries.get(alpha_2=kw["country_code"]) is not None:
+                return []
+        return subdivisions
+
+    def match(self, query):
+        query = remove_accents(query.strip().lower())
+        matching_candidates = []
+        for candidate in subdivisions:
+            for v in candidate._fields.values():
+                if v is not None:
+                    v = remove_accents(v.lower())
+                    # Some names include alternative versions which we want to
+                    # match exactly.
+                    for w in v.split(";"):
+                        if w == query:
+                            matching_candidates.append(candidate)
+                            break
+
+        return matching_candidates
+
+    def partial_match(self, query):
+        query = remove_accents(query.strip().lower())
+        matching_candidates = []
+        for candidate in subdivisions:
+            v = candidate._fields.get("name")
+            v = remove_accents(v.lower())
+            if query in v:
+                matching_candidates.append(candidate)
+
+        return matching_candidates
+
+    def search_fuzzy(self, query: str) -> List[Type["Subdivisions"]]:
+        query = remove_accents(query.strip().lower())
+
+        # A Subdivision's code to points mapping for later sorting subdivisions
+        # based on the query's matching incidence.
+        results: dict[str, int] = {}
+
+        def add_result(
+            subdivision: "pycountry.db.Subdivision", points: int
+        ) -> None:
+            results.setdefault(subdivision.code, 0)
+            results[subdivision.code] += points
+
+        # Prio 1: exact matches on subdivision names
+        match_subdivisions = self.match(query)
+        for candidate in match_subdivisions:
+            add_result(candidate, 50)
+
+        # Prio 2: partial matches on subdivision names
+        partial_match_subdivisions = self.partial_match(query)
+        for candidate in partial_match_subdivisions:
+            v = candidate._fields.get("name")
+            v = remove_accents(v.lower())
+            if query in v:
+                add_result(candidate, max([1, 5 - v.find(query)]))
+
+        if not results:
+            raise LookupError(query)
+
+        sorted_results = [
+            self.get(code=x[0])
+            # sort by points first, by alpha2 code second, and to ensure stable
+            # results the negative value allows us to sort reversely on the
+            # points but ascending on the country code.
+            for x in sorted(results.items(), key=lambda x: (-x[1], x[0]))
+        ]
+        return sorted_results
+
+
+# Initialize instances with type hints
+countries: ExistingCountries = ExistingCountries(
+    os.path.join(DATABASE_DIR, "iso3166-1.json")
+)
+subdivisions: Subdivisions = Subdivisions(
+    os.path.join(DATABASE_DIR, "iso3166-2.json")
+)
+historic_countries: HistoricCountries = HistoricCountries(
+    os.path.join(DATABASE_DIR, "iso3166-3.json")
+)
+
+currencies: Currencies = Currencies(os.path.join(DATABASE_DIR, "iso4217.json"))
+
+languages: Languages = Languages(os.path.join(DATABASE_DIR, "iso639-3.json"))
+language_families: LanguageFamilies = LanguageFamilies(
+    os.path.join(DATABASE_DIR, "iso639-5.json")
+)
+
+scripts: Scripts = Scripts(os.path.join(DATABASE_DIR, "iso15924.json"))
diff --git a/lib/python3.1/site-packages/pycountry/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/site-packages/pycountry/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..117b58d16d943e42a41faaf21d12e6ac44ab1358
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/__pycache__/__init__.cpython-310.pyc differ
diff --git a/lib/python3.1/site-packages/pycountry/__pycache__/db.cpython-310.pyc b/lib/python3.1/site-packages/pycountry/__pycache__/db.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..65476ec293fd336cb938e1cd7facb4c3c29d8449
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/__pycache__/db.cpython-310.pyc differ
diff --git a/lib/python3.1/site-packages/pycountry/databases/iso15924.json b/lib/python3.1/site-packages/pycountry/databases/iso15924.json
new file mode 100644
index 0000000000000000000000000000000000000000..32a31139ab6f5ea0aedb7e598d37ddc44a8ac37e
--- /dev/null
+++ b/lib/python3.1/site-packages/pycountry/databases/iso15924.json
@@ -0,0 +1,914 @@
+{
+  "15924": [
+    {
+      "alpha_4": "Adlm",
+      "name": "Adlam",
+      "numeric": "166"
+    },
+    {
+      "alpha_4": "Afak",
+      "name": "Afaka",
+      "numeric": "439"
+    },
+    {
+      "alpha_4": "Aghb",
+      "name": "Caucasian Albanian",
+      "numeric": "239"
+    },
+    {
+      "alpha_4": "Ahom",
+      "name": "Ahom, Tai Ahom",
+      "numeric": "338"
+    },
+    {
+      "alpha_4": "Arab",
+      "name": "Arabic",
+      "numeric": "160"
+    },
+    {
+      "alpha_4": "Aran",
+      "name": "Arabic (Nastaliq variant)",
+      "numeric": "161"
+    },
+    {
+      "alpha_4": "Armi",
+      "name": "Imperial Aramaic",
+      "numeric": "124"
+    },
+    {
+      "alpha_4": "Armn",
+      "name": "Armenian",
+      "numeric": "230"
+    },
+    {
+      "alpha_4": "Avst",
+      "name": "Avestan",
+      "numeric": "134"
+    },
+    {
+      "alpha_4": "Bali",
+      "name": "Balinese",
+      "numeric": "360"
+    },
+    {
+      "alpha_4": "Bamu",
+      "name": "Bamum",
+      "numeric": "435"
+    },
+    {
+      "alpha_4": "Bass",
+      "name": "Bassa Vah",
+      "numeric": "259"
+    },
+    {
+      "alpha_4": "Batk",
+      "name": "Batak",
+      "numeric": "365"
+    },
+    {
+      "alpha_4": "Beng",
+      "name": "Bengali",
+      "numeric": "325"
+    },
+    {
+      "alpha_4": "Bhks",
+      "name": "Bhaiksuki",
+      "numeric": "334"
+    },
+    {
+      "alpha_4": "Blis",
+      "name": "Blissymbols",
+      "numeric": "550"
+    },
+    {
+      "alpha_4": "Bopo",
+      "name": "Bopomofo",
+      "numeric": "285"
+    },
+    {
+      "alpha_4": "Brah",
+      "name": "Brahmi",
+      "numeric": "300"
+    },
+    {
+      "alpha_4": "Brai",
+      "name": "Braille",
+      "numeric": "570"
+    },
+    {
+      "alpha_4": "Bugi",
+      "name": "Buginese",
+      "numeric": "367"
+    },
+    {
+      "alpha_4": "Buhd",
+      "name": "Buhid",
+      "numeric": "372"
+    },
+    {
+      "alpha_4": "Cakm",
+      "name": "Chakma",
+      "numeric": "349"
+    },
+    {
+      "alpha_4": "Cans",
+      "name": "Unified Canadian Aboriginal Syllabics",
+      "numeric": "440"
+    },
+    {
+      "alpha_4": "Cari",
+      "name": "Carian",
+      "numeric": "201"
+    },
+    {
+      "alpha_4": "Cham",
+      "name": "Cham",
+      "numeric": "358"
+    },
+    {
+      "alpha_4": "Cher",
+      "name": "Cherokee",
+      "numeric": "445"
+    },
+    {
+      "alpha_4": "Cirt",
+      "name": "Cirth",
+      "numeric": "291"
+    },
+    {
+      "alpha_4": "Copt",
+      "name": "Coptic",
+      "numeric": "204"
+    },
+    {
+      "alpha_4": "Cprt",
+      "name": "Cypriot",
+      "numeric": "403"
+    },
+    {
+      "alpha_4": "Cyrl",
+      "name": "Cyrillic",
+      "numeric": "220"
+    },
+    {
+      "alpha_4": "Cyrs",
+      "name": "Cyrillic (Old Church Slavonic variant)",
+      "numeric": "221"
+    },
+    {
+      "alpha_4": "Deva",
+      "name": "Devanagari (Nagari)",
+      "numeric": "315"
+    },
+    {
+      "alpha_4": "Dsrt",
+      "name": "Deseret (Mormon)",
+      "numeric": "250"
+    },
+    {
+      "alpha_4": "Dupl",
+      "name": "Duployan shorthand, Duployan stenography",
+      "numeric": "755"
+    },
+    {
+      "alpha_4": "Egyd",
+      "name": "Egyptian demotic",
+      "numeric": "070"
+    },
+    {
+      "alpha_4": "Egyh",
+      "name": "Egyptian hieratic",
+      "numeric": "060"
+    },
+    {
+      "alpha_4": "Egyp",
+      "name": "Egyptian hieroglyphs",
+      "numeric": "050"
+    },
+    {
+      "alpha_4": "Elba",
+      "name": "Elbasan",
+      "numeric": "226"
+    },
+    {
+      "alpha_4": "Ethi",
+      "name": "Ethiopic (Geʻez)",
+      "numeric": "430"
+    },
+    {
+      "alpha_4": "Geok",
+      "name": "Khutsuri (Asomtavruli and Nuskhuri)",
+      "numeric": "241"
+    },
+    {
+      "alpha_4": "Geor",
+      "name": "Georgian (Mkhedruli)",
+      "numeric": "240"
+    },
+    {
+      "alpha_4": "Glag",
+      "name": "Glagolitic",
+      "numeric": "225"
+    },
+    {
+      "alpha_4": "Goth",
+      "name": "Gothic",
+      "numeric": "206"
+    },
+    {
+      "alpha_4": "Gran",
+      "name": "Grantha",
+      "numeric": "343"
+    },
+    {
+      "alpha_4": "Grek",
+      "name": "Greek",
+      "numeric": "200"
+    },
+    {
+      "alpha_4": "Gujr",
+      "name": "Gujarati",
+      "numeric": "320"
+    },
+    {
+      "alpha_4": "Guru",
+      "name": "Gurmukhi",
+      "numeric": "310"
+    },
+    {
+      "alpha_4": "Hanb",
+      "name": "Han with Bopomofo (alias for Han + Bopomofo)",
+      "numeric": "503"
+    },
+    {
+      "alpha_4": "Hang",
+      "name": "Hangul (Hangŭl, Hangeul)",
+      "numeric": "286"
+    },
+    {
+      "alpha_4": "Hani",
+      "name": "Han (Hanzi, Kanji, Hanja)",
+      "numeric": "500"
+    },
+    {
+      "alpha_4": "Hano",
+      "name": "Hanunoo (Hanunóo)",
+      "numeric": "371"
+    },
+    {
+      "alpha_4": "Hans",
+      "name": "Han (Simplified variant)",
+      "numeric": "501"
+    },
+    {
+      "alpha_4": "Hant",
+      "name": "Han (Traditional variant)",
+      "numeric": "502"
+    },
+    {
+      "alpha_4": "Hatr",
+      "name": "Hatran",
+      "numeric": "127"
+    },
+    {
+      "alpha_4": "Hebr",
+      "name": "Hebrew",
+      "numeric": "125"
+    },
+    {
+      "alpha_4": "Hira",
+      "name": "Hiragana",
+      "numeric": "410"
+    },
+    {
+      "alpha_4": "Hluw",
+      "name": "Anatolian Hieroglyphs (Luwian Hieroglyphs, Hittite Hieroglyphs)",
+      "numeric": "080"
+    },
+    {
+      "alpha_4": "Hmng",
+      "name": "Pahawh Hmong",
+      "numeric": "450"
+    },
+    {
+      "alpha_4": "Hrkt",
+      "name": "Japanese syllabaries (alias for Hiragana + Katakana)",
+      "numeric": "412"
+    },
+    {
+      "alpha_4": "Hung",
+      "name": "Old Hungarian (Hungarian Runic)",
+      "numeric": "176"
+    },
+    {
+      "alpha_4": "Inds",
+      "name": "Indus (Harappan)",
+      "numeric": "610"
+    },
+    {
+      "alpha_4": "Ital",
+      "name": "Old Italic (Etruscan, Oscan, etc.)",
+      "numeric": "210"
+    },
+    {
+      "alpha_4": "Jamo",
+      "name": "Jamo (alias for Jamo subset of Hangul)",
+      "numeric": "284"
+    },
+    {
+      "alpha_4": "Java",
+      "name": "Javanese",
+      "numeric": "361"
+    },
+    {
+      "alpha_4": "Jpan",
+      "name": "Japanese (alias for Han + Hiragana + Katakana)",
+      "numeric": "413"
+    },
+    {
+      "alpha_4": "Jurc",
+      "name": "Jurchen",
+      "numeric": "510"
+    },
+    {
+      "alpha_4": "Kali",
+      "name": "Kayah Li",
+      "numeric": "357"
+    },
+    {
+      "alpha_4": "Kana",
+      "name": "Katakana",
+      "numeric": "411"
+    },
+    {
+      "alpha_4": "Khar",
+      "name": "Kharoshthi",
+      "numeric": "305"
+    },
+    {
+      "alpha_4": "Khmr",
+      "name": "Khmer",
+      "numeric": "355"
+    },
+    {
+      "alpha_4": "Khoj",
+      "name": "Khojki",
+      "numeric": "322"
+    },
+    {
+      "alpha_4": "Kitl",
+      "name": "Khitan large script",
+      "numeric": "505"
+    },
+    {
+      "alpha_4": "Kits",
+      "name": "Khitan small script",
+      "numeric": "288"
+    },
+    {
+      "alpha_4": "Knda",
+      "name": "Kannada",
+      "numeric": "345"
+    },
+    {
+      "alpha_4": "Kore",
+      "name": "Korean (alias for Hangul + Han)",
+      "numeric": "287"
+    },
+    {
+      "alpha_4": "Kpel",
+      "name": "Kpelle",
+      "numeric": "436"
+    },
+    {
+      "alpha_4": "Kthi",
+      "name": "Kaithi",
+      "numeric": "317"
+    },
+    {
+      "alpha_4": "Lana",
+      "name": "Tai Tham (Lanna)",
+      "numeric": "351"
+    },
+    {
+      "alpha_4": "Laoo",
+      "name": "Lao",
+      "numeric": "356"
+    },
+    {
+      "alpha_4": "Latf",
+      "name": "Latin (Fraktur variant)",
+      "numeric": "217"
+    },
+    {
+      "alpha_4": "Latg",
+      "name": "Latin (Gaelic variant)",
+      "numeric": "216"
+    },
+    {
+      "alpha_4": "Latn",
+      "name": "Latin",
+      "numeric": "215"
+    },
+    {
+      "alpha_4": "Leke",
+      "name": "Leke",
+      "numeric": "364"
+    },
+    {
+      "alpha_4": "Lepc",
+      "name": "Lepcha (Róng)",
+      "numeric": "335"
+    },
+    {
+      "alpha_4": "Limb",
+      "name": "Limbu",
+      "numeric": "336"
+    },
+    {
+      "alpha_4": "Lina",
+      "name": "Linear A",
+      "numeric": "400"
+    },
+    {
+      "alpha_4": "Linb",
+      "name": "Linear B",
+      "numeric": "401"
+    },
+    {
+      "alpha_4": "Lisu",
+      "name": "Lisu (Fraser)",
+      "numeric": "399"
+    },
+    {
+      "alpha_4": "Loma",
+      "name": "Loma",
+      "numeric": "437"
+    },
+    {
+      "alpha_4": "Lyci",
+      "name": "Lycian",
+      "numeric": "202"
+    },
+    {
+      "alpha_4": "Lydi",
+      "name": "Lydian",
+      "numeric": "116"
+    },
+    {
+      "alpha_4": "Mahj",
+      "name": "Mahajani",
+      "numeric": "314"
+    },
+    {
+      "alpha_4": "Mand",
+      "name": "Mandaic, Mandaean",
+      "numeric": "140"
+    },
+    {
+      "alpha_4": "Mani",
+      "name": "Manichaean",
+      "numeric": "139"
+    },
+    {
+      "alpha_4": "Marc",
+      "name": "Marchen",
+      "numeric": "332"
+    },
+    {
+      "alpha_4": "Maya",
+      "name": "Mayan hieroglyphs",
+      "numeric": "090"
+    },
+    {
+      "alpha_4": "Mend",
+      "name": "Mende Kikakui",
+      "numeric": "438"
+    },
+    {
+      "alpha_4": "Merc",
+      "name": "Meroitic Cursive",
+      "numeric": "101"
+    },
+    {
+      "alpha_4": "Mero",
+      "name": "Meroitic Hieroglyphs",
+      "numeric": "100"
+    },
+    {
+      "alpha_4": "Mlym",
+      "name": "Malayalam",
+      "numeric": "347"
+    },
+    {
+      "alpha_4": "Modi",
+      "name": "Modi, Moḍī",
+      "numeric": "324"
+    },
+    {
+      "alpha_4": "Mong",
+      "name": "Mongolian",
+      "numeric": "145"
+    },
+    {
+      "alpha_4": "Moon",
+      "name": "Moon (Moon code, Moon script, Moon type)",
+      "numeric": "218"
+    },
+    {
+      "alpha_4": "Mroo",
+      "name": "Mro, Mru",
+      "numeric": "199"
+    },
+    {
+      "alpha_4": "Mtei",
+      "name": "Meitei Mayek (Meithei, Meetei)",
+      "numeric": "337"
+    },
+    {
+      "alpha_4": "Mult",
+      "name": "Multani",
+      "numeric": "323"
+    },
+    {
+      "alpha_4": "Mymr",
+      "name": "Myanmar (Burmese)",
+      "numeric": "350"
+    },
+    {
+      "alpha_4": "Narb",
+      "name": "Old North Arabian (Ancient North Arabian)",
+      "numeric": "106"
+    },
+    {
+      "alpha_4": "Nbat",
+      "name": "Nabataean",
+      "numeric": "159"
+    },
+    {
+      "alpha_4": "Newa",
+      "name": "Newa, Newar, Newari, Nepāla lipi",
+      "numeric": "333"
+    },
+    {
+      "alpha_4": "Nkgb",
+      "name": "Nakhi Geba ('Na-'Khi ²Ggŏ-¹baw, Naxi Geba)",
+      "numeric": "420"
+    },
+    {
+      "alpha_4": "Nkoo",
+      "name": "N’Ko",
+      "numeric": "165"
+    },
+    {
+      "alpha_4": "Nshu",
+      "name": "Nüshu",
+      "numeric": "499"
+    },
+    {
+      "alpha_4": "Ogam",
+      "name": "Ogham",
+      "numeric": "212"
+    },
+    {
+      "alpha_4": "Olck",
+      "name": "Ol Chiki (Ol Cemet’, Ol, Santali)",
+      "numeric": "261"
+    },
+    {
+      "alpha_4": "Orkh",
+      "name": "Old Turkic, Orkhon Runic",
+      "numeric": "175"
+    },
+    {
+      "alpha_4": "Orya",
+      "name": "Oriya",
+      "numeric": "327"
+    },
+    {
+      "alpha_4": "Osge",
+      "name": "Osage",
+      "numeric": "219"
+    },
+    {
+      "alpha_4": "Osma",
+      "name": "Osmanya",
+      "numeric": "260"
+    },
+    {
+      "alpha_4": "Palm",
+      "name": "Palmyrene",
+      "numeric": "126"
+    },
+    {
+      "alpha_4": "Pauc",
+      "name": "Pau Cin Hau",
+      "numeric": "263"
+    },
+    {
+      "alpha_4": "Perm",
+      "name": "Old Permic",
+      "numeric": "227"
+    },
+    {
+      "alpha_4": "Phag",
+      "name": "Phags-pa",
+      "numeric": "331"
+    },
+    {
+      "alpha_4": "Phli",
+      "name": "Inscriptional Pahlavi",
+      "numeric": "131"
+    },
+    {
+      "alpha_4": "Phlp",
+      "name": "Psalter Pahlavi",
+      "numeric": "132"
+    },
+    {
+      "alpha_4": "Phlv",
+      "name": "Book Pahlavi",
+      "numeric": "133"
+    },
+    {
+      "alpha_4": "Phnx",
+      "name": "Phoenician",
+      "numeric": "115"
+    },
+    {
+      "alpha_4": "Piqd",
+      "name": "Klingon (KLI pIqaD)",
+      "numeric": "293"
+    },
+    {
+      "alpha_4": "Plrd",
+      "name": "Miao (Pollard)",
+      "numeric": "282"
+    },
+    {
+      "alpha_4": "Prti",
+      "name": "Inscriptional Parthian",
+      "numeric": "130"
+    },
+    {
+      "alpha_4": "Qaaa",
+      "name": "Reserved for private use (start)",
+      "numeric": "900"
+    },
+    {
+      "alpha_4": "Qabx",
+      "name": "Reserved for private use (end)",
+      "numeric": "949"
+    },
+    {
+      "alpha_4": "Rjng",
+      "name": "Rejang (Redjang, Kaganga)",
+      "numeric": "363"
+    },
+    {
+      "alpha_4": "Roro",
+      "name": "Rongorongo",
+      "numeric": "620"
+    },
+    {
+      "alpha_4": "Runr",
+      "name": "Runic",
+      "numeric": "211"
+    },
+    {
+      "alpha_4": "Samr",
+      "name": "Samaritan",
+      "numeric": "123"
+    },
+    {
+      "alpha_4": "Sara",
+      "name": "Sarati",
+      "numeric": "292"
+    },
+    {
+      "alpha_4": "Sarb",
+      "name": "Old South Arabian",
+      "numeric": "105"
+    },
+    {
+      "alpha_4": "Saur",
+      "name": "Saurashtra",
+      "numeric": "344"
+    },
+    {
+      "alpha_4": "Sgnw",
+      "name": "SignWriting",
+      "numeric": "095"
+    },
+    {
+      "alpha_4": "Shaw",
+      "name": "Shavian (Shaw)",
+      "numeric": "281"
+    },
+    {
+      "alpha_4": "Shrd",
+      "name": "Sharada, Śāradā",
+      "numeric": "319"
+    },
+    {
+      "alpha_4": "Sidd",
+      "name": "Siddham, Siddhaṃ, Siddhamātṛkā",
+      "numeric": "302"
+    },
+    {
+      "alpha_4": "Sind",
+      "name": "Khudawadi, Sindhi",
+      "numeric": "318"
+    },
+    {
+      "alpha_4": "Sinh",
+      "name": "Sinhala",
+      "numeric": "348"
+    },
+    {
+      "alpha_4": "Sora",
+      "name": "Sora Sompeng",
+      "numeric": "398"
+    },
+    {
+      "alpha_4": "Sund",
+      "name": "Sundanese",
+      "numeric": "362"
+    },
+    {
+      "alpha_4": "Sylo",
+      "name": "Syloti Nagri",
+      "numeric": "316"
+    },
+    {
+      "alpha_4": "Syrc",
+      "name": "Syriac",
+      "numeric": "135"
+    },
+    {
+      "alpha_4": "Syre",
+      "name": "Syriac (Estrangelo variant)",
+      "numeric": "138"
+    },
+    {
+      "alpha_4": "Syrj",
+      "name": "Syriac (Western variant)",
+      "numeric": "137"
+    },
+    {
+      "alpha_4": "Syrn",
+      "name": "Syriac (Eastern variant)",
+      "numeric": "136"
+    },
+    {
+      "alpha_4": "Tagb",
+      "name": "Tagbanwa",
+      "numeric": "373"
+    },
+    {
+      "alpha_4": "Takr",
+      "name": "Takri, Ṭākrī, Ṭāṅkrī",
+      "numeric": "321"
+    },
+    {
+      "alpha_4": "Tale",
+      "name": "Tai Le",
+      "numeric": "353"
+    },
+    {
+      "alpha_4": "Talu",
+      "name": "New Tai Lue",
+      "numeric": "354"
+    },
+    {
+      "alpha_4": "Taml",
+      "name": "Tamil",
+      "numeric": "346"
+    },
+    {
+      "alpha_4": "Tang",
+      "name": "Tangut",
+      "numeric": "520"
+    },
+    {
+      "alpha_4": "Tavt",
+      "name": "Tai Viet",
+      "numeric": "359"
+    },
+    {
+      "alpha_4": "Telu",
+      "name": "Telugu",
+      "numeric": "340"
+    },
+    {
+      "alpha_4": "Teng",
+      "name": "Tengwar",
+      "numeric": "290"
+    },
+    {
+      "alpha_4": "Tfng",
+      "name": "Tifinagh (Berber)",
+      "numeric": "120"
+    },
+    {
+      "alpha_4": "Tglg",
+      "name": "Tagalog (Baybayin, Alibata)",
+      "numeric": "370"
+    },
+    {
+      "alpha_4": "Thaa",
+      "name": "Thaana",
+      "numeric": "170"
+    },
+    {
+      "alpha_4": "Thai",
+      "name": "Thai",
+      "numeric": "352"
+    },
+    {
+      "alpha_4": "Tibt",
+      "name": "Tibetan",
+      "numeric": "330"
+    },
+    {
+      "alpha_4": "Tirh",
+      "name": "Tirhuta",
+      "numeric": "326"
+    },
+    {
+      "alpha_4": "Ugar",
+      "name": "Ugaritic",
+      "numeric": "040"
+    },
+    {
+      "alpha_4": "Vaii",
+      "name": "Vai",
+      "numeric": "470"
+    },
+    {
+      "alpha_4": "Visp",
+      "name": "Visible Speech",
+      "numeric": "280"
+    },
+    {
+      "alpha_4": "Wara",
+      "name": "Warang Citi (Varang Kshiti)",
+      "numeric": "262"
+    },
+    {
+      "alpha_4": "Wole",
+      "name": "Woleai",
+      "numeric": "480"
+    },
+    {
+      "alpha_4": "Xpeo",
+      "name": "Old Persian",
+      "numeric": "030"
+    },
+    {
+      "alpha_4": "Xsux",
+      "name": "Cuneiform, Sumero-Akkadian",
+      "numeric": "020"
+    },
+    {
+      "alpha_4": "Yiii",
+      "name": "Yi",
+      "numeric": "460"
+    },
+    {
+      "alpha_4": "Zinh",
+      "name": "Code for inherited script",
+      "numeric": "994"
+    },
+    {
+      "alpha_4": "Zmth",
+      "name": "Mathematical notation",
+      "numeric": "995"
+    },
+    {
+      "alpha_4": "Zsye",
+      "name": "Symbols (Emoji variant)",
+      "numeric": "993"
+    },
+    {
+      "alpha_4": "Zsym",
+      "name": "Symbols",
+      "numeric": "996"
+    },
+    {
+      "alpha_4": "Zxxx",
+      "name": "Code for unwritten documents",
+      "numeric": "997"
+    },
+    {
+      "alpha_4": "Zyyy",
+      "name": "Code for undetermined script",
+      "numeric": "998"
+    },
+    {
+      "alpha_4": "Zzzz",
+      "name": "Code for uncoded script",
+      "numeric": "999"
+    }
+  ]
+}
diff --git a/lib/python3.1/site-packages/pycountry/databases/iso3166-1.json b/lib/python3.1/site-packages/pycountry/databases/iso3166-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..ee48a43ecb73808e5acbdc839cbe17f7cc32664b
--- /dev/null
+++ b/lib/python3.1/site-packages/pycountry/databases/iso3166-1.json
@@ -0,0 +1,1931 @@
+{
+  "3166-1": [
+    {
+      "alpha_2": "AW",
+      "alpha_3": "ABW",
+      "flag": "🇦🇼",
+      "name": "Aruba",
+      "numeric": "533"
+    },
+    {
+      "alpha_2": "AF",
+      "alpha_3": "AFG",
+      "flag": "🇦🇫",
+      "name": "Afghanistan",
+      "numeric": "004",
+      "official_name": "Islamic Republic of Afghanistan"
+    },
+    {
+      "alpha_2": "AO",
+      "alpha_3": "AGO",
+      "flag": "🇦🇴",
+      "name": "Angola",
+      "numeric": "024",
+      "official_name": "Republic of Angola"
+    },
+    {
+      "alpha_2": "AI",
+      "alpha_3": "AIA",
+      "flag": "🇦🇮",
+      "name": "Anguilla",
+      "numeric": "660"
+    },
+    {
+      "alpha_2": "AX",
+      "alpha_3": "ALA",
+      "flag": "🇦🇽",
+      "name": "Åland Islands",
+      "numeric": "248"
+    },
+    {
+      "alpha_2": "AL",
+      "alpha_3": "ALB",
+      "flag": "🇦🇱",
+      "name": "Albania",
+      "numeric": "008",
+      "official_name": "Republic of Albania"
+    },
+    {
+      "alpha_2": "AD",
+      "alpha_3": "AND",
+      "flag": "🇦🇩",
+      "name": "Andorra",
+      "numeric": "020",
+      "official_name": "Principality of Andorra"
+    },
+    {
+      "alpha_2": "AE",
+      "alpha_3": "ARE",
+      "flag": "🇦🇪",
+      "name": "United Arab Emirates",
+      "numeric": "784"
+    },
+    {
+      "alpha_2": "AR",
+      "alpha_3": "ARG",
+      "flag": "🇦🇷",
+      "name": "Argentina",
+      "numeric": "032",
+      "official_name": "Argentine Republic"
+    },
+    {
+      "alpha_2": "AM",
+      "alpha_3": "ARM",
+      "flag": "🇦🇲",
+      "name": "Armenia",
+      "numeric": "051",
+      "official_name": "Republic of Armenia"
+    },
+    {
+      "alpha_2": "AS",
+      "alpha_3": "ASM",
+      "flag": "🇦🇸",
+      "name": "American Samoa",
+      "numeric": "016"
+    },
+    {
+      "alpha_2": "AQ",
+      "alpha_3": "ATA",
+      "flag": "🇦🇶",
+      "name": "Antarctica",
+      "numeric": "010"
+    },
+    {
+      "alpha_2": "TF",
+      "alpha_3": "ATF",
+      "flag": "🇹🇫",
+      "name": "French Southern Territories",
+      "numeric": "260"
+    },
+    {
+      "alpha_2": "AG",
+      "alpha_3": "ATG",
+      "flag": "🇦🇬",
+      "name": "Antigua and Barbuda",
+      "numeric": "028"
+    },
+    {
+      "alpha_2": "AU",
+      "alpha_3": "AUS",
+      "flag": "🇦🇺",
+      "name": "Australia",
+      "numeric": "036"
+    },
+    {
+      "alpha_2": "AT",
+      "alpha_3": "AUT",
+      "flag": "🇦🇹",
+      "name": "Austria",
+      "numeric": "040",
+      "official_name": "Republic of Austria"
+    },
+    {
+      "alpha_2": "AZ",
+      "alpha_3": "AZE",
+      "flag": "🇦🇿",
+      "name": "Azerbaijan",
+      "numeric": "031",
+      "official_name": "Republic of Azerbaijan"
+    },
+    {
+      "alpha_2": "BI",
+      "alpha_3": "BDI",
+      "flag": "🇧🇮",
+      "name": "Burundi",
+      "numeric": "108",
+      "official_name": "Republic of Burundi"
+    },
+    {
+      "alpha_2": "BE",
+      "alpha_3": "BEL",
+      "flag": "🇧🇪",
+      "name": "Belgium",
+      "numeric": "056",
+      "official_name": "Kingdom of Belgium"
+    },
+    {
+      "alpha_2": "BJ",
+      "alpha_3": "BEN",
+      "flag": "🇧🇯",
+      "name": "Benin",
+      "numeric": "204",
+      "official_name": "Republic of Benin"
+    },
+    {
+      "alpha_2": "BQ",
+      "alpha_3": "BES",
+      "flag": "🇧🇶",
+      "name": "Bonaire, Sint Eustatius and Saba",
+      "numeric": "535",
+      "official_name": "Bonaire, Sint Eustatius and Saba"
+    },
+    {
+      "alpha_2": "BF",
+      "alpha_3": "BFA",
+      "flag": "🇧🇫",
+      "name": "Burkina Faso",
+      "numeric": "854"
+    },
+    {
+      "alpha_2": "BD",
+      "alpha_3": "BGD",
+      "flag": "🇧🇩",
+      "name": "Bangladesh",
+      "numeric": "050",
+      "official_name": "People's Republic of Bangladesh"
+    },
+    {
+      "alpha_2": "BG",
+      "alpha_3": "BGR",
+      "flag": "🇧🇬",
+      "name": "Bulgaria",
+      "numeric": "100",
+      "official_name": "Republic of Bulgaria"
+    },
+    {
+      "alpha_2": "BH",
+      "alpha_3": "BHR",
+      "flag": "🇧🇭",
+      "name": "Bahrain",
+      "numeric": "048",
+      "official_name": "Kingdom of Bahrain"
+    },
+    {
+      "alpha_2": "BS",
+      "alpha_3": "BHS",
+      "flag": "🇧🇸",
+      "name": "Bahamas",
+      "numeric": "044",
+      "official_name": "Commonwealth of the Bahamas"
+    },
+    {
+      "alpha_2": "BA",
+      "alpha_3": "BIH",
+      "flag": "🇧🇦",
+      "name": "Bosnia and Herzegovina",
+      "numeric": "070",
+      "official_name": "Republic of Bosnia and Herzegovina"
+    },
+    {
+      "alpha_2": "BL",
+      "alpha_3": "BLM",
+      "flag": "🇧🇱",
+      "name": "Saint Barthélemy",
+      "numeric": "652"
+    },
+    {
+      "alpha_2": "BY",
+      "alpha_3": "BLR",
+      "flag": "🇧🇾",
+      "name": "Belarus",
+      "numeric": "112",
+      "official_name": "Republic of Belarus"
+    },
+    {
+      "alpha_2": "BZ",
+      "alpha_3": "BLZ",
+      "flag": "🇧🇿",
+      "name": "Belize",
+      "numeric": "084"
+    },
+    {
+      "alpha_2": "BM",
+      "alpha_3": "BMU",
+      "flag": "🇧🇲",
+      "name": "Bermuda",
+      "numeric": "060"
+    },
+    {
+      "alpha_2": "BO",
+      "alpha_3": "BOL",
+      "common_name": "Bolivia",
+      "flag": "🇧🇴",
+      "name": "Bolivia, Plurinational State of",
+      "numeric": "068",
+      "official_name": "Plurinational State of Bolivia"
+    },
+    {
+      "alpha_2": "BR",
+      "alpha_3": "BRA",
+      "flag": "🇧🇷",
+      "name": "Brazil",
+      "numeric": "076",
+      "official_name": "Federative Republic of Brazil"
+    },
+    {
+      "alpha_2": "BB",
+      "alpha_3": "BRB",
+      "flag": "🇧🇧",
+      "name": "Barbados",
+      "numeric": "052"
+    },
+    {
+      "alpha_2": "BN",
+      "alpha_3": "BRN",
+      "flag": "🇧🇳",
+      "name": "Brunei Darussalam",
+      "numeric": "096"
+    },
+    {
+      "alpha_2": "BT",
+      "alpha_3": "BTN",
+      "flag": "🇧🇹",
+      "name": "Bhutan",
+      "numeric": "064",
+      "official_name": "Kingdom of Bhutan"
+    },
+    {
+      "alpha_2": "BV",
+      "alpha_3": "BVT",
+      "flag": "🇧🇻",
+      "name": "Bouvet Island",
+      "numeric": "074"
+    },
+    {
+      "alpha_2": "BW",
+      "alpha_3": "BWA",
+      "flag": "🇧🇼",
+      "name": "Botswana",
+      "numeric": "072",
+      "official_name": "Republic of Botswana"
+    },
+    {
+      "alpha_2": "CF",
+      "alpha_3": "CAF",
+      "flag": "🇨🇫",
+      "name": "Central African Republic",
+      "numeric": "140"
+    },
+    {
+      "alpha_2": "CA",
+      "alpha_3": "CAN",
+      "flag": "🇨🇦",
+      "name": "Canada",
+      "numeric": "124"
+    },
+    {
+      "alpha_2": "CC",
+      "alpha_3": "CCK",
+      "flag": "🇨🇨",
+      "name": "Cocos (Keeling) Islands",
+      "numeric": "166"
+    },
+    {
+      "alpha_2": "CH",
+      "alpha_3": "CHE",
+      "flag": "🇨🇭",
+      "name": "Switzerland",
+      "numeric": "756",
+      "official_name": "Swiss Confederation"
+    },
+    {
+      "alpha_2": "CL",
+      "alpha_3": "CHL",
+      "flag": "🇨🇱",
+      "name": "Chile",
+      "numeric": "152",
+      "official_name": "Republic of Chile"
+    },
+    {
+      "alpha_2": "CN",
+      "alpha_3": "CHN",
+      "flag": "🇨🇳",
+      "name": "China",
+      "numeric": "156",
+      "official_name": "People's Republic of China"
+    },
+    {
+      "alpha_2": "CI",
+      "alpha_3": "CIV",
+      "flag": "🇨🇮",
+      "name": "Côte d'Ivoire",
+      "numeric": "384",
+      "official_name": "Republic of Côte d'Ivoire"
+    },
+    {
+      "alpha_2": "CM",
+      "alpha_3": "CMR",
+      "flag": "🇨🇲",
+      "name": "Cameroon",
+      "numeric": "120",
+      "official_name": "Republic of Cameroon"
+    },
+    {
+      "alpha_2": "CD",
+      "alpha_3": "COD",
+      "flag": "🇨🇩",
+      "name": "Congo, The Democratic Republic of the",
+      "numeric": "180"
+    },
+    {
+      "alpha_2": "CG",
+      "alpha_3": "COG",
+      "flag": "🇨🇬",
+      "name": "Congo",
+      "numeric": "178",
+      "official_name": "Republic of the Congo"
+    },
+    {
+      "alpha_2": "CK",
+      "alpha_3": "COK",
+      "flag": "🇨🇰",
+      "name": "Cook Islands",
+      "numeric": "184"
+    },
+    {
+      "alpha_2": "CO",
+      "alpha_3": "COL",
+      "flag": "🇨🇴",
+      "name": "Colombia",
+      "numeric": "170",
+      "official_name": "Republic of Colombia"
+    },
+    {
+      "alpha_2": "KM",
+      "alpha_3": "COM",
+      "flag": "🇰🇲",
+      "name": "Comoros",
+      "numeric": "174",
+      "official_name": "Union of the Comoros"
+    },
+    {
+      "alpha_2": "CV",
+      "alpha_3": "CPV",
+      "flag": "🇨🇻",
+      "name": "Cabo Verde",
+      "numeric": "132",
+      "official_name": "Republic of Cabo Verde"
+    },
+    {
+      "alpha_2": "CR",
+      "alpha_3": "CRI",
+      "flag": "🇨🇷",
+      "name": "Costa Rica",
+      "numeric": "188",
+      "official_name": "Republic of Costa Rica"
+    },
+    {
+      "alpha_2": "CU",
+      "alpha_3": "CUB",
+      "flag": "🇨🇺",
+      "name": "Cuba",
+      "numeric": "192",
+      "official_name": "Republic of Cuba"
+    },
+    {
+      "alpha_2": "CW",
+      "alpha_3": "CUW",
+      "flag": "🇨🇼",
+      "name": "Curaçao",
+      "numeric": "531",
+      "official_name": "Curaçao"
+    },
+    {
+      "alpha_2": "CX",
+      "alpha_3": "CXR",
+      "flag": "🇨🇽",
+      "name": "Christmas Island",
+      "numeric": "162"
+    },
+    {
+      "alpha_2": "KY",
+      "alpha_3": "CYM",
+      "flag": "🇰🇾",
+      "name": "Cayman Islands",
+      "numeric": "136"
+    },
+    {
+      "alpha_2": "CY",
+      "alpha_3": "CYP",
+      "flag": "🇨🇾",
+      "name": "Cyprus",
+      "numeric": "196",
+      "official_name": "Republic of Cyprus"
+    },
+    {
+      "alpha_2": "CZ",
+      "alpha_3": "CZE",
+      "flag": "🇨🇿",
+      "name": "Czechia",
+      "numeric": "203",
+      "official_name": "Czech Republic"
+    },
+    {
+      "alpha_2": "DE",
+      "alpha_3": "DEU",
+      "flag": "🇩🇪",
+      "name": "Germany",
+      "numeric": "276",
+      "official_name": "Federal Republic of Germany"
+    },
+    {
+      "alpha_2": "DJ",
+      "alpha_3": "DJI",
+      "flag": "🇩🇯",
+      "name": "Djibouti",
+      "numeric": "262",
+      "official_name": "Republic of Djibouti"
+    },
+    {
+      "alpha_2": "DM",
+      "alpha_3": "DMA",
+      "flag": "🇩🇲",
+      "name": "Dominica",
+      "numeric": "212",
+      "official_name": "Commonwealth of Dominica"
+    },
+    {
+      "alpha_2": "DK",
+      "alpha_3": "DNK",
+      "flag": "🇩🇰",
+      "name": "Denmark",
+      "numeric": "208",
+      "official_name": "Kingdom of Denmark"
+    },
+    {
+      "alpha_2": "DO",
+      "alpha_3": "DOM",
+      "flag": "🇩🇴",
+      "name": "Dominican Republic",
+      "numeric": "214"
+    },
+    {
+      "alpha_2": "DZ",
+      "alpha_3": "DZA",
+      "flag": "🇩🇿",
+      "name": "Algeria",
+      "numeric": "012",
+      "official_name": "People's Democratic Republic of Algeria"
+    },
+    {
+      "alpha_2": "EC",
+      "alpha_3": "ECU",
+      "flag": "🇪🇨",
+      "name": "Ecuador",
+      "numeric": "218",
+      "official_name": "Republic of Ecuador"
+    },
+    {
+      "alpha_2": "EG",
+      "alpha_3": "EGY",
+      "flag": "🇪🇬",
+      "name": "Egypt",
+      "numeric": "818",
+      "official_name": "Arab Republic of Egypt"
+    },
+    {
+      "alpha_2": "ER",
+      "alpha_3": "ERI",
+      "flag": "🇪🇷",
+      "name": "Eritrea",
+      "numeric": "232",
+      "official_name": "the State of Eritrea"
+    },
+    {
+      "alpha_2": "EH",
+      "alpha_3": "ESH",
+      "flag": "🇪🇭",
+      "name": "Western Sahara",
+      "numeric": "732"
+    },
+    {
+      "alpha_2": "ES",
+      "alpha_3": "ESP",
+      "flag": "🇪🇸",
+      "name": "Spain",
+      "numeric": "724",
+      "official_name": "Kingdom of Spain"
+    },
+    {
+      "alpha_2": "EE",
+      "alpha_3": "EST",
+      "flag": "🇪🇪",
+      "name": "Estonia",
+      "numeric": "233",
+      "official_name": "Republic of Estonia"
+    },
+    {
+      "alpha_2": "ET",
+      "alpha_3": "ETH",
+      "flag": "🇪🇹",
+      "name": "Ethiopia",
+      "numeric": "231",
+      "official_name": "Federal Democratic Republic of Ethiopia"
+    },
+    {
+      "alpha_2": "FI",
+      "alpha_3": "FIN",
+      "flag": "🇫🇮",
+      "name": "Finland",
+      "numeric": "246",
+      "official_name": "Republic of Finland"
+    },
+    {
+      "alpha_2": "FJ",
+      "alpha_3": "FJI",
+      "flag": "🇫🇯",
+      "name": "Fiji",
+      "numeric": "242",
+      "official_name": "Republic of Fiji"
+    },
+    {
+      "alpha_2": "FK",
+      "alpha_3": "FLK",
+      "flag": "🇫🇰",
+      "name": "Falkland Islands (Malvinas)",
+      "numeric": "238"
+    },
+    {
+      "alpha_2": "FR",
+      "alpha_3": "FRA",
+      "flag": "🇫🇷",
+      "name": "France",
+      "numeric": "250",
+      "official_name": "French Republic"
+    },
+    {
+      "alpha_2": "FO",
+      "alpha_3": "FRO",
+      "flag": "🇫🇴",
+      "name": "Faroe Islands",
+      "numeric": "234"
+    },
+    {
+      "alpha_2": "FM",
+      "alpha_3": "FSM",
+      "flag": "🇫🇲",
+      "name": "Micronesia, Federated States of",
+      "numeric": "583",
+      "official_name": "Federated States of Micronesia"
+    },
+    {
+      "alpha_2": "GA",
+      "alpha_3": "GAB",
+      "flag": "🇬🇦",
+      "name": "Gabon",
+      "numeric": "266",
+      "official_name": "Gabonese Republic"
+    },
+    {
+      "alpha_2": "GB",
+      "alpha_3": "GBR",
+      "flag": "🇬🇧",
+      "name": "United Kingdom",
+      "numeric": "826",
+      "official_name": "United Kingdom of Great Britain and Northern Ireland"
+    },
+    {
+      "alpha_2": "GE",
+      "alpha_3": "GEO",
+      "flag": "🇬🇪",
+      "name": "Georgia",
+      "numeric": "268"
+    },
+    {
+      "alpha_2": "GG",
+      "alpha_3": "GGY",
+      "flag": "🇬🇬",
+      "name": "Guernsey",
+      "numeric": "831"
+    },
+    {
+      "alpha_2": "GH",
+      "alpha_3": "GHA",
+      "flag": "🇬🇭",
+      "name": "Ghana",
+      "numeric": "288",
+      "official_name": "Republic of Ghana"
+    },
+    {
+      "alpha_2": "GI",
+      "alpha_3": "GIB",
+      "flag": "🇬🇮",
+      "name": "Gibraltar",
+      "numeric": "292"
+    },
+    {
+      "alpha_2": "GN",
+      "alpha_3": "GIN",
+      "flag": "🇬🇳",
+      "name": "Guinea",
+      "numeric": "324",
+      "official_name": "Republic of Guinea"
+    },
+    {
+      "alpha_2": "GP",
+      "alpha_3": "GLP",
+      "flag": "🇬🇵",
+      "name": "Guadeloupe",
+      "numeric": "312"
+    },
+    {
+      "alpha_2": "GM",
+      "alpha_3": "GMB",
+      "flag": "🇬🇲",
+      "name": "Gambia",
+      "numeric": "270",
+      "official_name": "Republic of the Gambia"
+    },
+    {
+      "alpha_2": "GW",
+      "alpha_3": "GNB",
+      "flag": "🇬🇼",
+      "name": "Guinea-Bissau",
+      "numeric": "624",
+      "official_name": "Republic of Guinea-Bissau"
+    },
+    {
+      "alpha_2": "GQ",
+      "alpha_3": "GNQ",
+      "flag": "🇬🇶",
+      "name": "Equatorial Guinea",
+      "numeric": "226",
+      "official_name": "Republic of Equatorial Guinea"
+    },
+    {
+      "alpha_2": "GR",
+      "alpha_3": "GRC",
+      "flag": "🇬🇷",
+      "name": "Greece",
+      "numeric": "300",
+      "official_name": "Hellenic Republic"
+    },
+    {
+      "alpha_2": "GD",
+      "alpha_3": "GRD",
+      "flag": "🇬🇩",
+      "name": "Grenada",
+      "numeric": "308"
+    },
+    {
+      "alpha_2": "GL",
+      "alpha_3": "GRL",
+      "flag": "🇬🇱",
+      "name": "Greenland",
+      "numeric": "304"
+    },
+    {
+      "alpha_2": "GT",
+      "alpha_3": "GTM",
+      "flag": "🇬🇹",
+      "name": "Guatemala",
+      "numeric": "320",
+      "official_name": "Republic of Guatemala"
+    },
+    {
+      "alpha_2": "GF",
+      "alpha_3": "GUF",
+      "flag": "🇬🇫",
+      "name": "French Guiana",
+      "numeric": "254"
+    },
+    {
+      "alpha_2": "GU",
+      "alpha_3": "GUM",
+      "flag": "🇬🇺",
+      "name": "Guam",
+      "numeric": "316"
+    },
+    {
+      "alpha_2": "GY",
+      "alpha_3": "GUY",
+      "flag": "🇬🇾",
+      "name": "Guyana",
+      "numeric": "328",
+      "official_name": "Republic of Guyana"
+    },
+    {
+      "alpha_2": "HK",
+      "alpha_3": "HKG",
+      "flag": "🇭🇰",
+      "name": "Hong Kong",
+      "numeric": "344",
+      "official_name": "Hong Kong Special Administrative Region of China"
+    },
+    {
+      "alpha_2": "HM",
+      "alpha_3": "HMD",
+      "flag": "🇭🇲",
+      "name": "Heard Island and McDonald Islands",
+      "numeric": "334"
+    },
+    {
+      "alpha_2": "HN",
+      "alpha_3": "HND",
+      "flag": "🇭🇳",
+      "name": "Honduras",
+      "numeric": "340",
+      "official_name": "Republic of Honduras"
+    },
+    {
+      "alpha_2": "HR",
+      "alpha_3": "HRV",
+      "flag": "🇭🇷",
+      "name": "Croatia",
+      "numeric": "191",
+      "official_name": "Republic of Croatia"
+    },
+    {
+      "alpha_2": "HT",
+      "alpha_3": "HTI",
+      "flag": "🇭🇹",
+      "name": "Haiti",
+      "numeric": "332",
+      "official_name": "Republic of Haiti"
+    },
+    {
+      "alpha_2": "HU",
+      "alpha_3": "HUN",
+      "flag": "🇭🇺",
+      "name": "Hungary",
+      "numeric": "348",
+      "official_name": "Hungary"
+    },
+    {
+      "alpha_2": "ID",
+      "alpha_3": "IDN",
+      "flag": "🇮🇩",
+      "name": "Indonesia",
+      "numeric": "360",
+      "official_name": "Republic of Indonesia"
+    },
+    {
+      "alpha_2": "IM",
+      "alpha_3": "IMN",
+      "flag": "🇮🇲",
+      "name": "Isle of Man",
+      "numeric": "833"
+    },
+    {
+      "alpha_2": "IN",
+      "alpha_3": "IND",
+      "flag": "🇮🇳",
+      "name": "India",
+      "numeric": "356",
+      "official_name": "Republic of India"
+    },
+    {
+      "alpha_2": "IO",
+      "alpha_3": "IOT",
+      "flag": "🇮🇴",
+      "name": "British Indian Ocean Territory",
+      "numeric": "086"
+    },
+    {
+      "alpha_2": "IE",
+      "alpha_3": "IRL",
+      "flag": "🇮🇪",
+      "name": "Ireland",
+      "numeric": "372"
+    },
+    {
+      "alpha_2": "IR",
+      "alpha_3": "IRN",
+      "common_name": "Iran",
+      "flag": "🇮🇷",
+      "name": "Iran, Islamic Republic of",
+      "numeric": "364",
+      "official_name": "Islamic Republic of Iran"
+    },
+    {
+      "alpha_2": "IQ",
+      "alpha_3": "IRQ",
+      "flag": "🇮🇶",
+      "name": "Iraq",
+      "numeric": "368",
+      "official_name": "Republic of Iraq"
+    },
+    {
+      "alpha_2": "IS",
+      "alpha_3": "ISL",
+      "flag": "🇮🇸",
+      "name": "Iceland",
+      "numeric": "352",
+      "official_name": "Republic of Iceland"
+    },
+    {
+      "alpha_2": "IL",
+      "alpha_3": "ISR",
+      "flag": "🇮🇱",
+      "name": "Israel",
+      "numeric": "376",
+      "official_name": "State of Israel"
+    },
+    {
+      "alpha_2": "IT",
+      "alpha_3": "ITA",
+      "flag": "🇮🇹",
+      "name": "Italy",
+      "numeric": "380",
+      "official_name": "Italian Republic"
+    },
+    {
+      "alpha_2": "JM",
+      "alpha_3": "JAM",
+      "flag": "🇯🇲",
+      "name": "Jamaica",
+      "numeric": "388"
+    },
+    {
+      "alpha_2": "JE",
+      "alpha_3": "JEY",
+      "flag": "🇯🇪",
+      "name": "Jersey",
+      "numeric": "832"
+    },
+    {
+      "alpha_2": "JO",
+      "alpha_3": "JOR",
+      "flag": "🇯🇴",
+      "name": "Jordan",
+      "numeric": "400",
+      "official_name": "Hashemite Kingdom of Jordan"
+    },
+    {
+      "alpha_2": "JP",
+      "alpha_3": "JPN",
+      "flag": "🇯🇵",
+      "name": "Japan",
+      "numeric": "392"
+    },
+    {
+      "alpha_2": "KZ",
+      "alpha_3": "KAZ",
+      "flag": "🇰🇿",
+      "name": "Kazakhstan",
+      "numeric": "398",
+      "official_name": "Republic of Kazakhstan"
+    },
+    {
+      "alpha_2": "KE",
+      "alpha_3": "KEN",
+      "flag": "🇰🇪",
+      "name": "Kenya",
+      "numeric": "404",
+      "official_name": "Republic of Kenya"
+    },
+    {
+      "alpha_2": "KG",
+      "alpha_3": "KGZ",
+      "flag": "🇰🇬",
+      "name": "Kyrgyzstan",
+      "numeric": "417",
+      "official_name": "Kyrgyz Republic"
+    },
+    {
+      "alpha_2": "KH",
+      "alpha_3": "KHM",
+      "flag": "🇰🇭",
+      "name": "Cambodia",
+      "numeric": "116",
+      "official_name": "Kingdom of Cambodia"
+    },
+    {
+      "alpha_2": "KI",
+      "alpha_3": "KIR",
+      "flag": "🇰🇮",
+      "name": "Kiribati",
+      "numeric": "296",
+      "official_name": "Republic of Kiribati"
+    },
+    {
+      "alpha_2": "KN",
+      "alpha_3": "KNA",
+      "flag": "🇰🇳",
+      "name": "Saint Kitts and Nevis",
+      "numeric": "659"
+    },
+    {
+      "alpha_2": "KR",
+      "alpha_3": "KOR",
+      "common_name": "South Korea",
+      "flag": "🇰🇷",
+      "name": "Korea, Republic of",
+      "numeric": "410"
+    },
+    {
+      "alpha_2": "KW",
+      "alpha_3": "KWT",
+      "flag": "🇰🇼",
+      "name": "Kuwait",
+      "numeric": "414",
+      "official_name": "State of Kuwait"
+    },
+    {
+      "alpha_2": "LA",
+      "alpha_3": "LAO",
+      "common_name": "Laos",
+      "flag": "🇱🇦",
+      "name": "Lao People's Democratic Republic",
+      "numeric": "418"
+    },
+    {
+      "alpha_2": "LB",
+      "alpha_3": "LBN",
+      "flag": "🇱🇧",
+      "name": "Lebanon",
+      "numeric": "422",
+      "official_name": "Lebanese Republic"
+    },
+    {
+      "alpha_2": "LR",
+      "alpha_3": "LBR",
+      "flag": "🇱🇷",
+      "name": "Liberia",
+      "numeric": "430",
+      "official_name": "Republic of Liberia"
+    },
+    {
+      "alpha_2": "LY",
+      "alpha_3": "LBY",
+      "flag": "🇱🇾",
+      "name": "Libya",
+      "numeric": "434",
+      "official_name": "Libya"
+    },
+    {
+      "alpha_2": "LC",
+      "alpha_3": "LCA",
+      "flag": "🇱🇨",
+      "name": "Saint Lucia",
+      "numeric": "662"
+    },
+    {
+      "alpha_2": "LI",
+      "alpha_3": "LIE",
+      "flag": "🇱🇮",
+      "name": "Liechtenstein",
+      "numeric": "438",
+      "official_name": "Principality of Liechtenstein"
+    },
+    {
+      "alpha_2": "LK",
+      "alpha_3": "LKA",
+      "flag": "🇱🇰",
+      "name": "Sri Lanka",
+      "numeric": "144",
+      "official_name": "Democratic Socialist Republic of Sri Lanka"
+    },
+    {
+      "alpha_2": "LS",
+      "alpha_3": "LSO",
+      "flag": "🇱🇸",
+      "name": "Lesotho",
+      "numeric": "426",
+      "official_name": "Kingdom of Lesotho"
+    },
+    {
+      "alpha_2": "LT",
+      "alpha_3": "LTU",
+      "flag": "🇱🇹",
+      "name": "Lithuania",
+      "numeric": "440",
+      "official_name": "Republic of Lithuania"
+    },
+    {
+      "alpha_2": "LU",
+      "alpha_3": "LUX",
+      "flag": "🇱🇺",
+      "name": "Luxembourg",
+      "numeric": "442",
+      "official_name": "Grand Duchy of Luxembourg"
+    },
+    {
+      "alpha_2": "LV",
+      "alpha_3": "LVA",
+      "flag": "🇱🇻",
+      "name": "Latvia",
+      "numeric": "428",
+      "official_name": "Republic of Latvia"
+    },
+    {
+      "alpha_2": "MO",
+      "alpha_3": "MAC",
+      "flag": "🇲🇴",
+      "name": "Macao",
+      "numeric": "446",
+      "official_name": "Macao Special Administrative Region of China"
+    },
+    {
+      "alpha_2": "MF",
+      "alpha_3": "MAF",
+      "flag": "🇲🇫",
+      "name": "Saint Martin (French part)",
+      "numeric": "663"
+    },
+    {
+      "alpha_2": "MA",
+      "alpha_3": "MAR",
+      "flag": "🇲🇦",
+      "name": "Morocco",
+      "numeric": "504",
+      "official_name": "Kingdom of Morocco"
+    },
+    {
+      "alpha_2": "MC",
+      "alpha_3": "MCO",
+      "flag": "🇲🇨",
+      "name": "Monaco",
+      "numeric": "492",
+      "official_name": "Principality of Monaco"
+    },
+    {
+      "alpha_2": "MD",
+      "alpha_3": "MDA",
+      "common_name": "Moldova",
+      "flag": "🇲🇩",
+      "name": "Moldova, Republic of",
+      "numeric": "498",
+      "official_name": "Republic of Moldova"
+    },
+    {
+      "alpha_2": "MG",
+      "alpha_3": "MDG",
+      "flag": "🇲🇬",
+      "name": "Madagascar",
+      "numeric": "450",
+      "official_name": "Republic of Madagascar"
+    },
+    {
+      "alpha_2": "MV",
+      "alpha_3": "MDV",
+      "flag": "🇲🇻",
+      "name": "Maldives",
+      "numeric": "462",
+      "official_name": "Republic of Maldives"
+    },
+    {
+      "alpha_2": "MX",
+      "alpha_3": "MEX",
+      "flag": "🇲🇽",
+      "name": "Mexico",
+      "numeric": "484",
+      "official_name": "United Mexican States"
+    },
+    {
+      "alpha_2": "MH",
+      "alpha_3": "MHL",
+      "flag": "🇲🇭",
+      "name": "Marshall Islands",
+      "numeric": "584",
+      "official_name": "Republic of the Marshall Islands"
+    },
+    {
+      "alpha_2": "MK",
+      "alpha_3": "MKD",
+      "flag": "🇲🇰",
+      "name": "North Macedonia",
+      "numeric": "807",
+      "official_name": "Republic of North Macedonia"
+    },
+    {
+      "alpha_2": "ML",
+      "alpha_3": "MLI",
+      "flag": "🇲🇱",
+      "name": "Mali",
+      "numeric": "466",
+      "official_name": "Republic of Mali"
+    },
+    {
+      "alpha_2": "MT",
+      "alpha_3": "MLT",
+      "flag": "🇲🇹",
+      "name": "Malta",
+      "numeric": "470",
+      "official_name": "Republic of Malta"
+    },
+    {
+      "alpha_2": "MM",
+      "alpha_3": "MMR",
+      "flag": "🇲🇲",
+      "name": "Myanmar",
+      "numeric": "104",
+      "official_name": "Republic of Myanmar"
+    },
+    {
+      "alpha_2": "ME",
+      "alpha_3": "MNE",
+      "flag": "🇲🇪",
+      "name": "Montenegro",
+      "numeric": "499",
+      "official_name": "Montenegro"
+    },
+    {
+      "alpha_2": "MN",
+      "alpha_3": "MNG",
+      "flag": "🇲🇳",
+      "name": "Mongolia",
+      "numeric": "496"
+    },
+    {
+      "alpha_2": "MP",
+      "alpha_3": "MNP",
+      "flag": "🇲🇵",
+      "name": "Northern Mariana Islands",
+      "numeric": "580",
+      "official_name": "Commonwealth of the Northern Mariana Islands"
+    },
+    {
+      "alpha_2": "MZ",
+      "alpha_3": "MOZ",
+      "flag": "🇲🇿",
+      "name": "Mozambique",
+      "numeric": "508",
+      "official_name": "Republic of Mozambique"
+    },
+    {
+      "alpha_2": "MR",
+      "alpha_3": "MRT",
+      "flag": "🇲🇷",
+      "name": "Mauritania",
+      "numeric": "478",
+      "official_name": "Islamic Republic of Mauritania"
+    },
+    {
+      "alpha_2": "MS",
+      "alpha_3": "MSR",
+      "flag": "🇲🇸",
+      "name": "Montserrat",
+      "numeric": "500"
+    },
+    {
+      "alpha_2": "MQ",
+      "alpha_3": "MTQ",
+      "flag": "🇲🇶",
+      "name": "Martinique",
+      "numeric": "474"
+    },
+    {
+      "alpha_2": "MU",
+      "alpha_3": "MUS",
+      "flag": "🇲🇺",
+      "name": "Mauritius",
+      "numeric": "480",
+      "official_name": "Republic of Mauritius"
+    },
+    {
+      "alpha_2": "MW",
+      "alpha_3": "MWI",
+      "flag": "🇲🇼",
+      "name": "Malawi",
+      "numeric": "454",
+      "official_name": "Republic of Malawi"
+    },
+    {
+      "alpha_2": "MY",
+      "alpha_3": "MYS",
+      "flag": "🇲🇾",
+      "name": "Malaysia",
+      "numeric": "458"
+    },
+    {
+      "alpha_2": "YT",
+      "alpha_3": "MYT",
+      "flag": "🇾🇹",
+      "name": "Mayotte",
+      "numeric": "175"
+    },
+    {
+      "alpha_2": "NA",
+      "alpha_3": "NAM",
+      "flag": "🇳🇦",
+      "name": "Namibia",
+      "numeric": "516",
+      "official_name": "Republic of Namibia"
+    },
+    {
+      "alpha_2": "NC",
+      "alpha_3": "NCL",
+      "flag": "🇳🇨",
+      "name": "New Caledonia",
+      "numeric": "540"
+    },
+    {
+      "alpha_2": "NE",
+      "alpha_3": "NER",
+      "flag": "🇳🇪",
+      "name": "Niger",
+      "numeric": "562",
+      "official_name": "Republic of the Niger"
+    },
+    {
+      "alpha_2": "NF",
+      "alpha_3": "NFK",
+      "flag": "🇳🇫",
+      "name": "Norfolk Island",
+      "numeric": "574"
+    },
+    {
+      "alpha_2": "NG",
+      "alpha_3": "NGA",
+      "flag": "🇳🇬",
+      "name": "Nigeria",
+      "numeric": "566",
+      "official_name": "Federal Republic of Nigeria"
+    },
+    {
+      "alpha_2": "NI",
+      "alpha_3": "NIC",
+      "flag": "🇳🇮",
+      "name": "Nicaragua",
+      "numeric": "558",
+      "official_name": "Republic of Nicaragua"
+    },
+    {
+      "alpha_2": "NU",
+      "alpha_3": "NIU",
+      "flag": "🇳🇺",
+      "name": "Niue",
+      "numeric": "570",
+      "official_name": "Niue"
+    },
+    {
+      "alpha_2": "NL",
+      "alpha_3": "NLD",
+      "flag": "🇳🇱",
+      "name": "Netherlands",
+      "numeric": "528",
+      "official_name": "Kingdom of the Netherlands"
+    },
+    {
+      "alpha_2": "NO",
+      "alpha_3": "NOR",
+      "flag": "🇳🇴",
+      "name": "Norway",
+      "numeric": "578",
+      "official_name": "Kingdom of Norway"
+    },
+    {
+      "alpha_2": "NP",
+      "alpha_3": "NPL",
+      "flag": "🇳🇵",
+      "name": "Nepal",
+      "numeric": "524",
+      "official_name": "Federal Democratic Republic of Nepal"
+    },
+    {
+      "alpha_2": "NR",
+      "alpha_3": "NRU",
+      "flag": "🇳🇷",
+      "name": "Nauru",
+      "numeric": "520",
+      "official_name": "Republic of Nauru"
+    },
+    {
+      "alpha_2": "NZ",
+      "alpha_3": "NZL",
+      "flag": "🇳🇿",
+      "name": "New Zealand",
+      "numeric": "554"
+    },
+    {
+      "alpha_2": "OM",
+      "alpha_3": "OMN",
+      "flag": "🇴🇲",
+      "name": "Oman",
+      "numeric": "512",
+      "official_name": "Sultanate of Oman"
+    },
+    {
+      "alpha_2": "PK",
+      "alpha_3": "PAK",
+      "flag": "🇵🇰",
+      "name": "Pakistan",
+      "numeric": "586",
+      "official_name": "Islamic Republic of Pakistan"
+    },
+    {
+      "alpha_2": "PA",
+      "alpha_3": "PAN",
+      "flag": "🇵🇦",
+      "name": "Panama",
+      "numeric": "591",
+      "official_name": "Republic of Panama"
+    },
+    {
+      "alpha_2": "PN",
+      "alpha_3": "PCN",
+      "flag": "🇵🇳",
+      "name": "Pitcairn",
+      "numeric": "612"
+    },
+    {
+      "alpha_2": "PE",
+      "alpha_3": "PER",
+      "flag": "🇵🇪",
+      "name": "Peru",
+      "numeric": "604",
+      "official_name": "Republic of Peru"
+    },
+    {
+      "alpha_2": "PH",
+      "alpha_3": "PHL",
+      "flag": "🇵🇭",
+      "name": "Philippines",
+      "numeric": "608",
+      "official_name": "Republic of the Philippines"
+    },
+    {
+      "alpha_2": "PW",
+      "alpha_3": "PLW",
+      "flag": "🇵🇼",
+      "name": "Palau",
+      "numeric": "585",
+      "official_name": "Republic of Palau"
+    },
+    {
+      "alpha_2": "PG",
+      "alpha_3": "PNG",
+      "flag": "🇵🇬",
+      "name": "Papua New Guinea",
+      "numeric": "598",
+      "official_name": "Independent State of Papua New Guinea"
+    },
+    {
+      "alpha_2": "PL",
+      "alpha_3": "POL",
+      "flag": "🇵🇱",
+      "name": "Poland",
+      "numeric": "616",
+      "official_name": "Republic of Poland"
+    },
+    {
+      "alpha_2": "PR",
+      "alpha_3": "PRI",
+      "flag": "🇵🇷",
+      "name": "Puerto Rico",
+      "numeric": "630"
+    },
+    {
+      "alpha_2": "KP",
+      "alpha_3": "PRK",
+      "common_name": "North Korea",
+      "flag": "🇰🇵",
+      "name": "Korea, Democratic People's Republic of",
+      "numeric": "408",
+      "official_name": "Democratic People's Republic of Korea"
+    },
+    {
+      "alpha_2": "PT",
+      "alpha_3": "PRT",
+      "flag": "🇵🇹",
+      "name": "Portugal",
+      "numeric": "620",
+      "official_name": "Portuguese Republic"
+    },
+    {
+      "alpha_2": "PY",
+      "alpha_3": "PRY",
+      "flag": "🇵🇾",
+      "name": "Paraguay",
+      "numeric": "600",
+      "official_name": "Republic of Paraguay"
+    },
+    {
+      "alpha_2": "PS",
+      "alpha_3": "PSE",
+      "flag": "🇵🇸",
+      "name": "Palestine, State of",
+      "numeric": "275",
+      "official_name": "the State of Palestine"
+    },
+    {
+      "alpha_2": "PF",
+      "alpha_3": "PYF",
+      "flag": "🇵🇫",
+      "name": "French Polynesia",
+      "numeric": "258"
+    },
+    {
+      "alpha_2": "QA",
+      "alpha_3": "QAT",
+      "flag": "🇶🇦",
+      "name": "Qatar",
+      "numeric": "634",
+      "official_name": "State of Qatar"
+    },
+    {
+      "alpha_2": "RE",
+      "alpha_3": "REU",
+      "flag": "🇷🇪",
+      "name": "Réunion",
+      "numeric": "638"
+    },
+    {
+      "alpha_2": "RO",
+      "alpha_3": "ROU",
+      "flag": "🇷🇴",
+      "name": "Romania",
+      "numeric": "642"
+    },
+    {
+      "alpha_2": "RU",
+      "alpha_3": "RUS",
+      "flag": "🇷🇺",
+      "name": "Russian Federation",
+      "numeric": "643"
+    },
+    {
+      "alpha_2": "RW",
+      "alpha_3": "RWA",
+      "flag": "🇷🇼",
+      "name": "Rwanda",
+      "numeric": "646",
+      "official_name": "Rwandese Republic"
+    },
+    {
+      "alpha_2": "SA",
+      "alpha_3": "SAU",
+      "flag": "🇸🇦",
+      "name": "Saudi Arabia",
+      "numeric": "682",
+      "official_name": "Kingdom of Saudi Arabia"
+    },
+    {
+      "alpha_2": "SD",
+      "alpha_3": "SDN",
+      "flag": "🇸🇩",
+      "name": "Sudan",
+      "numeric": "729",
+      "official_name": "Republic of the Sudan"
+    },
+    {
+      "alpha_2": "SN",
+      "alpha_3": "SEN",
+      "flag": "🇸🇳",
+      "name": "Senegal",
+      "numeric": "686",
+      "official_name": "Republic of Senegal"
+    },
+    {
+      "alpha_2": "SG",
+      "alpha_3": "SGP",
+      "flag": "🇸🇬",
+      "name": "Singapore",
+      "numeric": "702",
+      "official_name": "Republic of Singapore"
+    },
+    {
+      "alpha_2": "GS",
+      "alpha_3": "SGS",
+      "flag": "🇬🇸",
+      "name": "South Georgia and the South Sandwich Islands",
+      "numeric": "239"
+    },
+    {
+      "alpha_2": "SH",
+      "alpha_3": "SHN",
+      "flag": "🇸🇭",
+      "name": "Saint Helena, Ascension and Tristan da Cunha",
+      "numeric": "654"
+    },
+    {
+      "alpha_2": "SJ",
+      "alpha_3": "SJM",
+      "flag": "🇸🇯",
+      "name": "Svalbard and Jan Mayen",
+      "numeric": "744"
+    },
+    {
+      "alpha_2": "SB",
+      "alpha_3": "SLB",
+      "flag": "🇸🇧",
+      "name": "Solomon Islands",
+      "numeric": "090"
+    },
+    {
+      "alpha_2": "SL",
+      "alpha_3": "SLE",
+      "flag": "🇸🇱",
+      "name": "Sierra Leone",
+      "numeric": "694",
+      "official_name": "Republic of Sierra Leone"
+    },
+    {
+      "alpha_2": "SV",
+      "alpha_3": "SLV",
+      "flag": "🇸🇻",
+      "name": "El Salvador",
+      "numeric": "222",
+      "official_name": "Republic of El Salvador"
+    },
+    {
+      "alpha_2": "SM",
+      "alpha_3": "SMR",
+      "flag": "🇸🇲",
+      "name": "San Marino",
+      "numeric": "674",
+      "official_name": "Republic of San Marino"
+    },
+    {
+      "alpha_2": "SO",
+      "alpha_3": "SOM",
+      "flag": "🇸🇴",
+      "name": "Somalia",
+      "numeric": "706",
+      "official_name": "Federal Republic of Somalia"
+    },
+    {
+      "alpha_2": "PM",
+      "alpha_3": "SPM",
+      "flag": "🇵🇲",
+      "name": "Saint Pierre and Miquelon",
+      "numeric": "666"
+    },
+    {
+      "alpha_2": "RS",
+      "alpha_3": "SRB",
+      "flag": "🇷🇸",
+      "name": "Serbia",
+      "numeric": "688",
+      "official_name": "Republic of Serbia"
+    },
+    {
+      "alpha_2": "SS",
+      "alpha_3": "SSD",
+      "flag": "🇸🇸",
+      "name": "South Sudan",
+      "numeric": "728",
+      "official_name": "Republic of South Sudan"
+    },
+    {
+      "alpha_2": "ST",
+      "alpha_3": "STP",
+      "flag": "🇸🇹",
+      "name": "Sao Tome and Principe",
+      "numeric": "678",
+      "official_name": "Democratic Republic of Sao Tome and Principe"
+    },
+    {
+      "alpha_2": "SR",
+      "alpha_3": "SUR",
+      "flag": "🇸🇷",
+      "name": "Suriname",
+      "numeric": "740",
+      "official_name": "Republic of Suriname"
+    },
+    {
+      "alpha_2": "SK",
+      "alpha_3": "SVK",
+      "flag": "🇸🇰",
+      "name": "Slovakia",
+      "numeric": "703",
+      "official_name": "Slovak Republic"
+    },
+    {
+      "alpha_2": "SI",
+      "alpha_3": "SVN",
+      "flag": "🇸🇮",
+      "name": "Slovenia",
+      "numeric": "705",
+      "official_name": "Republic of Slovenia"
+    },
+    {
+      "alpha_2": "SE",
+      "alpha_3": "SWE",
+      "flag": "🇸🇪",
+      "name": "Sweden",
+      "numeric": "752",
+      "official_name": "Kingdom of Sweden"
+    },
+    {
+      "alpha_2": "SZ",
+      "alpha_3": "SWZ",
+      "flag": "🇸🇿",
+      "name": "Eswatini",
+      "numeric": "748",
+      "official_name": "Kingdom of Eswatini"
+    },
+    {
+      "alpha_2": "SX",
+      "alpha_3": "SXM",
+      "flag": "🇸🇽",
+      "name": "Sint Maarten (Dutch part)",
+      "numeric": "534",
+      "official_name": "Sint Maarten (Dutch part)"
+    },
+    {
+      "alpha_2": "SC",
+      "alpha_3": "SYC",
+      "flag": "🇸🇨",
+      "name": "Seychelles",
+      "numeric": "690",
+      "official_name": "Republic of Seychelles"
+    },
+    {
+      "alpha_2": "SY",
+      "alpha_3": "SYR",
+      "common_name": "Syria",
+      "flag": "🇸🇾",
+      "name": "Syrian Arab Republic",
+      "numeric": "760"
+    },
+    {
+      "alpha_2": "TC",
+      "alpha_3": "TCA",
+      "flag": "🇹🇨",
+      "name": "Turks and Caicos Islands",
+      "numeric": "796"
+    },
+    {
+      "alpha_2": "TD",
+      "alpha_3": "TCD",
+      "flag": "🇹🇩",
+      "name": "Chad",
+      "numeric": "148",
+      "official_name": "Republic of Chad"
+    },
+    {
+      "alpha_2": "TG",
+      "alpha_3": "TGO",
+      "flag": "🇹🇬",
+      "name": "Togo",
+      "numeric": "768",
+      "official_name": "Togolese Republic"
+    },
+    {
+      "alpha_2": "TH",
+      "alpha_3": "THA",
+      "flag": "🇹🇭",
+      "name": "Thailand",
+      "numeric": "764",
+      "official_name": "Kingdom of Thailand"
+    },
+    {
+      "alpha_2": "TJ",
+      "alpha_3": "TJK",
+      "flag": "🇹🇯",
+      "name": "Tajikistan",
+      "numeric": "762",
+      "official_name": "Republic of Tajikistan"
+    },
+    {
+      "alpha_2": "TK",
+      "alpha_3": "TKL",
+      "flag": "🇹🇰",
+      "name": "Tokelau",
+      "numeric": "772"
+    },
+    {
+      "alpha_2": "TM",
+      "alpha_3": "TKM",
+      "flag": "🇹🇲",
+      "name": "Turkmenistan",
+      "numeric": "795"
+    },
+    {
+      "alpha_2": "TL",
+      "alpha_3": "TLS",
+      "flag": "🇹🇱",
+      "name": "Timor-Leste",
+      "numeric": "626",
+      "official_name": "Democratic Republic of Timor-Leste"
+    },
+    {
+      "alpha_2": "TO",
+      "alpha_3": "TON",
+      "flag": "🇹🇴",
+      "name": "Tonga",
+      "numeric": "776",
+      "official_name": "Kingdom of Tonga"
+    },
+    {
+      "alpha_2": "TT",
+      "alpha_3": "TTO",
+      "flag": "🇹🇹",
+      "name": "Trinidad and Tobago",
+      "numeric": "780",
+      "official_name": "Republic of Trinidad and Tobago"
+    },
+    {
+      "alpha_2": "TN",
+      "alpha_3": "TUN",
+      "flag": "🇹🇳",
+      "name": "Tunisia",
+      "numeric": "788",
+      "official_name": "Republic of Tunisia"
+    },
+    {
+      "alpha_2": "TR",
+      "alpha_3": "TUR",
+      "flag": "🇹🇷",
+      "name": "Türkiye",
+      "numeric": "792",
+      "official_name": "Republic of Türkiye"
+    },
+    {
+      "alpha_2": "TV",
+      "alpha_3": "TUV",
+      "flag": "🇹🇻",
+      "name": "Tuvalu",
+      "numeric": "798"
+    },
+    {
+      "alpha_2": "TW",
+      "alpha_3": "TWN",
+      "common_name": "Taiwan",
+      "flag": "🇹🇼",
+      "name": "Taiwan, Province of China",
+      "numeric": "158",
+      "official_name": "Taiwan, Province of China"
+    },
+    {
+      "alpha_2": "TZ",
+      "alpha_3": "TZA",
+      "common_name": "Tanzania",
+      "flag": "🇹🇿",
+      "name": "Tanzania, United Republic of",
+      "numeric": "834",
+      "official_name": "United Republic of Tanzania"
+    },
+    {
+      "alpha_2": "UG",
+      "alpha_3": "UGA",
+      "flag": "🇺🇬",
+      "name": "Uganda",
+      "numeric": "800",
+      "official_name": "Republic of Uganda"
+    },
+    {
+      "alpha_2": "UA",
+      "alpha_3": "UKR",
+      "flag": "🇺🇦",
+      "name": "Ukraine",
+      "numeric": "804"
+    },
+    {
+      "alpha_2": "UM",
+      "alpha_3": "UMI",
+      "flag": "🇺🇲",
+      "name": "United States Minor Outlying Islands",
+      "numeric": "581"
+    },
+    {
+      "alpha_2": "UY",
+      "alpha_3": "URY",
+      "flag": "🇺🇾",
+      "name": "Uruguay",
+      "numeric": "858",
+      "official_name": "Eastern Republic of Uruguay"
+    },
+    {
+      "alpha_2": "US",
+      "alpha_3": "USA",
+      "flag": "🇺🇸",
+      "name": "United States",
+      "numeric": "840",
+      "official_name": "United States of America"
+    },
+    {
+      "alpha_2": "UZ",
+      "alpha_3": "UZB",
+      "flag": "🇺🇿",
+      "name": "Uzbekistan",
+      "numeric": "860",
+      "official_name": "Republic of Uzbekistan"
+    },
+    {
+      "alpha_2": "VA",
+      "alpha_3": "VAT",
+      "flag": "🇻🇦",
+      "name": "Holy See (Vatican City State)",
+      "numeric": "336"
+    },
+    {
+      "alpha_2": "VC",
+      "alpha_3": "VCT",
+      "flag": "🇻🇨",
+      "name": "Saint Vincent and the Grenadines",
+      "numeric": "670"
+    },
+    {
+      "alpha_2": "VE",
+      "alpha_3": "VEN",
+      "common_name": "Venezuela",
+      "flag": "🇻🇪",
+      "name": "Venezuela, Bolivarian Republic of",
+      "numeric": "862",
+      "official_name": "Bolivarian Republic of Venezuela"
+    },
+    {
+      "alpha_2": "VG",
+      "alpha_3": "VGB",
+      "flag": "🇻🇬",
+      "name": "Virgin Islands, British",
+      "numeric": "092",
+      "official_name": "British Virgin Islands"
+    },
+    {
+      "alpha_2": "VI",
+      "alpha_3": "VIR",
+      "flag": "🇻🇮",
+      "name": "Virgin Islands, U.S.",
+      "numeric": "850",
+      "official_name": "Virgin Islands of the United States"
+    },
+    {
+      "alpha_2": "VN",
+      "alpha_3": "VNM",
+      "common_name": "Vietnam",
+      "flag": "🇻🇳",
+      "name": "Viet Nam",
+      "numeric": "704",
+      "official_name": "Socialist Republic of Viet Nam"
+    },
+    {
+      "alpha_2": "VU",
+      "alpha_3": "VUT",
+      "flag": "🇻🇺",
+      "name": "Vanuatu",
+      "numeric": "548",
+      "official_name": "Republic of Vanuatu"
+    },
+    {
+      "alpha_2": "WF",
+      "alpha_3": "WLF",
+      "flag": "🇼🇫",
+      "name": "Wallis and Futuna",
+      "numeric": "876"
+    },
+    {
+      "alpha_2": "WS",
+      "alpha_3": "WSM",
+      "flag": "🇼🇸",
+      "name": "Samoa",
+      "numeric": "882",
+      "official_name": "Independent State of Samoa"
+    },
+    {
+      "alpha_2": "YE",
+      "alpha_3": "YEM",
+      "flag": "🇾🇪",
+      "name": "Yemen",
+      "numeric": "887",
+      "official_name": "Republic of Yemen"
+    },
+    {
+      "alpha_2": "ZA",
+      "alpha_3": "ZAF",
+      "flag": "🇿🇦",
+      "name": "South Africa",
+      "numeric": "710",
+      "official_name": "Republic of South Africa"
+    },
+    {
+      "alpha_2": "ZM",
+      "alpha_3": "ZMB",
+      "flag": "🇿🇲",
+      "name": "Zambia",
+      "numeric": "894",
+      "official_name": "Republic of Zambia"
+    },
+    {
+      "alpha_2": "ZW",
+      "alpha_3": "ZWE",
+      "flag": "🇿🇼",
+      "name": "Zimbabwe",
+      "numeric": "716",
+      "official_name": "Republic of Zimbabwe"
+    }
+  ]
+}
diff --git a/lib/python3.1/site-packages/pycountry/databases/iso3166-2.json b/lib/python3.1/site-packages/pycountry/databases/iso3166-2.json
new file mode 100644
index 0000000000000000000000000000000000000000..e16038b9a0cccdd5a879767044c77cacb774296b
--- /dev/null
+++ b/lib/python3.1/site-packages/pycountry/databases/iso3166-2.json
@@ -0,0 +1,26690 @@
+{
+  "3166-2": [
+    {
+      "code": "AD-02",
+      "name": "Canillo",
+      "type": "Parish"
+    },
+    {
+      "code": "AD-03",
+      "name": "Encamp",
+      "type": "Parish"
+    },
+    {
+      "code": "AD-04",
+      "name": "La Massana",
+      "type": "Parish"
+    },
+    {
+      "code": "AD-05",
+      "name": "Ordino",
+      "type": "Parish"
+    },
+    {
+      "code": "AD-06",
+      "name": "Sant Julià de Lòria",
+      "type": "Parish"
+    },
+    {
+      "code": "AD-07",
+      "name": "Andorra la Vella",
+      "type": "Parish"
+    },
+    {
+      "code": "AD-08",
+      "name": "Escaldes-Engordany",
+      "type": "Parish"
+    },
+    {
+      "code": "AE-AJ",
+      "name": "‘Ajmān",
+      "type": "Emirate"
+    },
+    {
+      "code": "AE-AZ",
+      "name": "Abū Z̧aby",
+      "type": "Emirate"
+    },
+    {
+      "code": "AE-DU",
+      "name": "Dubayy",
+      "type": "Emirate"
+    },
+    {
+      "code": "AE-FU",
+      "name": "Al Fujayrah",
+      "type": "Emirate"
+    },
+    {
+      "code": "AE-RK",
+      "name": "Ra’s al Khaymah",
+      "type": "Emirate"
+    },
+    {
+      "code": "AE-SH",
+      "name": "Ash Shāriqah",
+      "type": "Emirate"
+    },
+    {
+      "code": "AE-UQ",
+      "name": "Umm al Qaywayn",
+      "type": "Emirate"
+    },
+    {
+      "code": "AF-BAL",
+      "name": "Balkh",
+      "type": "Province"
+    },
+    {
+      "code": "AF-BAM",
+      "name": "Bāmyān",
+      "type": "Province"
+    },
+    {
+      "code": "AF-BDG",
+      "name": "Bādghīs",
+      "type": "Province"
+    },
+    {
+      "code": "AF-BDS",
+      "name": "Badakhshān",
+      "type": "Province"
+    },
+    {
+      "code": "AF-BGL",
+      "name": "Baghlān",
+      "type": "Province"
+    },
+    {
+      "code": "AF-DAY",
+      "name": "Dāykundī",
+      "type": "Province"
+    },
+    {
+      "code": "AF-FRA",
+      "name": "Farāh",
+      "type": "Province"
+    },
+    {
+      "code": "AF-FYB",
+      "name": "Fāryāb",
+      "type": "Province"
+    },
+    {
+      "code": "AF-GHA",
+      "name": "Ghaznī",
+      "type": "Province"
+    },
+    {
+      "code": "AF-GHO",
+      "name": "Ghōr",
+      "type": "Province"
+    },
+    {
+      "code": "AF-HEL",
+      "name": "Helmand",
+      "type": "Province"
+    },
+    {
+      "code": "AF-HER",
+      "name": "Herāt",
+      "type": "Province"
+    },
+    {
+      "code": "AF-JOW",
+      "name": "Jowzjān",
+      "type": "Province"
+    },
+    {
+      "code": "AF-KAB",
+      "name": "Kābul",
+      "type": "Province"
+    },
+    {
+      "code": "AF-KAN",
+      "name": "Kandahār",
+      "type": "Province"
+    },
+    {
+      "code": "AF-KAP",
+      "name": "Kāpīsā",
+      "type": "Province"
+    },
+    {
+      "code": "AF-KDZ",
+      "name": "Kunduz",
+      "type": "Province"
+    },
+    {
+      "code": "AF-KHO",
+      "name": "Khōst",
+      "type": "Province"
+    },
+    {
+      "code": "AF-KNR",
+      "name": "Kunaṟ",
+      "type": "Province"
+    },
+    {
+      "code": "AF-LAG",
+      "name": "Laghmān",
+      "type": "Province"
+    },
+    {
+      "code": "AF-LOG",
+      "name": "Lōgar",
+      "type": "Province"
+    },
+    {
+      "code": "AF-NAN",
+      "name": "Nangarhār",
+      "type": "Province"
+    },
+    {
+      "code": "AF-NIM",
+      "name": "Nīmrōz",
+      "type": "Province"
+    },
+    {
+      "code": "AF-NUR",
+      "name": "Nūristān",
+      "type": "Province"
+    },
+    {
+      "code": "AF-PAN",
+      "name": "Panjshayr",
+      "type": "Province"
+    },
+    {
+      "code": "AF-PAR",
+      "name": "Parwān",
+      "type": "Province"
+    },
+    {
+      "code": "AF-PIA",
+      "name": "Paktiyā",
+      "type": "Province"
+    },
+    {
+      "code": "AF-PKA",
+      "name": "Paktīkā",
+      "type": "Province"
+    },
+    {
+      "code": "AF-SAM",
+      "name": "Samangān",
+      "type": "Province"
+    },
+    {
+      "code": "AF-SAR",
+      "name": "Sar-e Pul",
+      "type": "Province"
+    },
+    {
+      "code": "AF-TAK",
+      "name": "Takhār",
+      "type": "Province"
+    },
+    {
+      "code": "AF-URU",
+      "name": "Uruzgān",
+      "type": "Province"
+    },
+    {
+      "code": "AF-WAR",
+      "name": "Wardak",
+      "type": "Province"
+    },
+    {
+      "code": "AF-ZAB",
+      "name": "Zābul",
+      "type": "Province"
+    },
+    {
+      "code": "AG-03",
+      "name": "Saint George",
+      "type": "Parish"
+    },
+    {
+      "code": "AG-04",
+      "name": "Saint John",
+      "type": "Parish"
+    },
+    {
+      "code": "AG-05",
+      "name": "Saint Mary",
+      "type": "Parish"
+    },
+    {
+      "code": "AG-06",
+      "name": "Saint Paul",
+      "type": "Parish"
+    },
+    {
+      "code": "AG-07",
+      "name": "Saint Peter",
+      "type": "Parish"
+    },
+    {
+      "code": "AG-08",
+      "name": "Saint Philip",
+      "type": "Parish"
+    },
+    {
+      "code": "AG-10",
+      "name": "Barbuda",
+      "type": "Dependency"
+    },
+    {
+      "code": "AG-11",
+      "name": "Redonda",
+      "type": "Dependency"
+    },
+    {
+      "code": "AL-01",
+      "name": "Berat",
+      "type": "County"
+    },
+    {
+      "code": "AL-02",
+      "name": "Durrës",
+      "type": "County"
+    },
+    {
+      "code": "AL-03",
+      "name": "Elbasan",
+      "type": "County"
+    },
+    {
+      "code": "AL-04",
+      "name": "Fier",
+      "type": "County"
+    },
+    {
+      "code": "AL-05",
+      "name": "Gjirokastër",
+      "type": "County"
+    },
+    {
+      "code": "AL-06",
+      "name": "Korçë",
+      "type": "County"
+    },
+    {
+      "code": "AL-07",
+      "name": "Kukës",
+      "type": "County"
+    },
+    {
+      "code": "AL-08",
+      "name": "Lezhë",
+      "type": "County"
+    },
+    {
+      "code": "AL-09",
+      "name": "Dibër",
+      "type": "County"
+    },
+    {
+      "code": "AL-10",
+      "name": "Shkodër",
+      "type": "County"
+    },
+    {
+      "code": "AL-11",
+      "name": "Tiranë",
+      "type": "County"
+    },
+    {
+      "code": "AL-12",
+      "name": "Vlorë",
+      "type": "County"
+    },
+    {
+      "code": "AM-AG",
+      "name": "Aragac̣otn",
+      "type": "Region"
+    },
+    {
+      "code": "AM-AR",
+      "name": "Ararat",
+      "type": "Region"
+    },
+    {
+      "code": "AM-AV",
+      "name": "Armavir",
+      "type": "Region"
+    },
+    {
+      "code": "AM-ER",
+      "name": "Erevan",
+      "type": "City"
+    },
+    {
+      "code": "AM-GR",
+      "name": "Geġark'unik'",
+      "type": "Region"
+    },
+    {
+      "code": "AM-KT",
+      "name": "Kotayk'",
+      "type": "Region"
+    },
+    {
+      "code": "AM-LO",
+      "name": "Loṙi",
+      "type": "Region"
+    },
+    {
+      "code": "AM-SH",
+      "name": "Širak",
+      "type": "Region"
+    },
+    {
+      "code": "AM-SU",
+      "name": "Syunik'",
+      "type": "Region"
+    },
+    {
+      "code": "AM-TV",
+      "name": "Tavuš",
+      "type": "Region"
+    },
+    {
+      "code": "AM-VD",
+      "name": "Vayoć Jor",
+      "type": "Region"
+    },
+    {
+      "code": "AO-BGO",
+      "name": "Bengo",
+      "type": "Province"
+    },
+    {
+      "code": "AO-BGU",
+      "name": "Benguela",
+      "type": "Province"
+    },
+    {
+      "code": "AO-BIE",
+      "name": "Bié",
+      "type": "Province"
+    },
+    {
+      "code": "AO-CAB",
+      "name": "Cabinda",
+      "type": "Province"
+    },
+    {
+      "code": "AO-CCU",
+      "name": "Cuando Cubango",
+      "type": "Province"
+    },
+    {
+      "code": "AO-CNN",
+      "name": "Cunene",
+      "type": "Province"
+    },
+    {
+      "code": "AO-CNO",
+      "name": "Cuanza-Norte",
+      "type": "Province"
+    },
+    {
+      "code": "AO-CUS",
+      "name": "Cuanza-Sul",
+      "type": "Province"
+    },
+    {
+      "code": "AO-HUA",
+      "name": "Huambo",
+      "type": "Province"
+    },
+    {
+      "code": "AO-HUI",
+      "name": "Huíla",
+      "type": "Province"
+    },
+    {
+      "code": "AO-LNO",
+      "name": "Lunda-Norte",
+      "type": "Province"
+    },
+    {
+      "code": "AO-LSU",
+      "name": "Lunda-Sul",
+      "type": "Province"
+    },
+    {
+      "code": "AO-LUA",
+      "name": "Luanda",
+      "type": "Province"
+    },
+    {
+      "code": "AO-MAL",
+      "name": "Malange",
+      "type": "Province"
+    },
+    {
+      "code": "AO-MOX",
+      "name": "Moxico",
+      "type": "Province"
+    },
+    {
+      "code": "AO-NAM",
+      "name": "Namibe",
+      "type": "Province"
+    },
+    {
+      "code": "AO-UIG",
+      "name": "Uíge",
+      "type": "Province"
+    },
+    {
+      "code": "AO-ZAI",
+      "name": "Zaire",
+      "type": "Province"
+    },
+    {
+      "code": "AR-A",
+      "name": "Salta",
+      "type": "Province"
+    },
+    {
+      "code": "AR-B",
+      "name": "Buenos Aires",
+      "type": "Province"
+    },
+    {
+      "code": "AR-C",
+      "name": "Ciudad Autónoma de Buenos Aires",
+      "type": "City"
+    },
+    {
+      "code": "AR-D",
+      "name": "San Luis",
+      "type": "Province"
+    },
+    {
+      "code": "AR-E",
+      "name": "Entre Ríos",
+      "type": "Province"
+    },
+    {
+      "code": "AR-F",
+      "name": "La Rioja",
+      "type": "Province"
+    },
+    {
+      "code": "AR-G",
+      "name": "Santiago del Estero",
+      "type": "Province"
+    },
+    {
+      "code": "AR-H",
+      "name": "Chaco",
+      "type": "Province"
+    },
+    {
+      "code": "AR-J",
+      "name": "San Juan",
+      "type": "Province"
+    },
+    {
+      "code": "AR-K",
+      "name": "Catamarca",
+      "type": "Province"
+    },
+    {
+      "code": "AR-L",
+      "name": "La Pampa",
+      "type": "Province"
+    },
+    {
+      "code": "AR-M",
+      "name": "Mendoza",
+      "type": "Province"
+    },
+    {
+      "code": "AR-N",
+      "name": "Misiones",
+      "type": "Province"
+    },
+    {
+      "code": "AR-P",
+      "name": "Formosa",
+      "type": "Province"
+    },
+    {
+      "code": "AR-Q",
+      "name": "Neuquén",
+      "type": "Province"
+    },
+    {
+      "code": "AR-R",
+      "name": "Río Negro",
+      "type": "Province"
+    },
+    {
+      "code": "AR-S",
+      "name": "Santa Fe",
+      "type": "Province"
+    },
+    {
+      "code": "AR-T",
+      "name": "Tucumán",
+      "type": "Province"
+    },
+    {
+      "code": "AR-U",
+      "name": "Chubut",
+      "type": "Province"
+    },
+    {
+      "code": "AR-V",
+      "name": "Tierra del Fuego",
+      "type": "Province"
+    },
+    {
+      "code": "AR-W",
+      "name": "Corrientes",
+      "type": "Province"
+    },
+    {
+      "code": "AR-X",
+      "name": "Córdoba",
+      "type": "Province"
+    },
+    {
+      "code": "AR-Y",
+      "name": "Jujuy",
+      "type": "Province"
+    },
+    {
+      "code": "AR-Z",
+      "name": "Santa Cruz",
+      "type": "Province"
+    },
+    {
+      "code": "AT-1",
+      "name": "Burgenland",
+      "type": "State"
+    },
+    {
+      "code": "AT-2",
+      "name": "Kärnten",
+      "type": "State"
+    },
+    {
+      "code": "AT-3",
+      "name": "Niederösterreich",
+      "type": "State"
+    },
+    {
+      "code": "AT-4",
+      "name": "Oberösterreich",
+      "type": "State"
+    },
+    {
+      "code": "AT-5",
+      "name": "Salzburg",
+      "type": "State"
+    },
+    {
+      "code": "AT-6",
+      "name": "Steiermark",
+      "type": "State"
+    },
+    {
+      "code": "AT-7",
+      "name": "Tirol",
+      "type": "State"
+    },
+    {
+      "code": "AT-8",
+      "name": "Vorarlberg",
+      "type": "State"
+    },
+    {
+      "code": "AT-9",
+      "name": "Wien",
+      "type": "State"
+    },
+    {
+      "code": "AU-ACT",
+      "name": "Australian Capital Territory",
+      "type": "Territory"
+    },
+    {
+      "code": "AU-NSW",
+      "name": "New South Wales",
+      "type": "State"
+    },
+    {
+      "code": "AU-NT",
+      "name": "Northern Territory",
+      "type": "Territory"
+    },
+    {
+      "code": "AU-QLD",
+      "name": "Queensland",
+      "type": "State"
+    },
+    {
+      "code": "AU-SA",
+      "name": "South Australia",
+      "type": "State"
+    },
+    {
+      "code": "AU-TAS",
+      "name": "Tasmania",
+      "type": "State"
+    },
+    {
+      "code": "AU-VIC",
+      "name": "Victoria",
+      "type": "State"
+    },
+    {
+      "code": "AU-WA",
+      "name": "Western Australia",
+      "type": "State"
+    },
+    {
+      "code": "AZ-ABS",
+      "name": "Abşeron",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-AGA",
+      "name": "Ağstafa",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-AGC",
+      "name": "Ağcabədi",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-AGM",
+      "name": "Ağdam",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-AGS",
+      "name": "Ağdaş",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-AGU",
+      "name": "Ağsu",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-AST",
+      "name": "Astara",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-BA",
+      "name": "Bakı",
+      "type": "Municipality"
+    },
+    {
+      "code": "AZ-BAB",
+      "name": "Babək",
+      "parent": "AZ-NX",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-BAL",
+      "name": "Balakən",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-BAR",
+      "name": "Bərdə",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-BEY",
+      "name": "Beyləqan",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-BIL",
+      "name": "Biləsuvar",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-CAB",
+      "name": "Cəbrayıl",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-CAL",
+      "name": "Cəlilabad",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-CUL",
+      "name": "Culfa",
+      "parent": "AZ-NX",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-DAS",
+      "name": "Daşkəsən",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-FUZ",
+      "name": "Füzuli",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-GA",
+      "name": "Gəncə",
+      "type": "Municipality"
+    },
+    {
+      "code": "AZ-GAD",
+      "name": "Gədəbəy",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-GOR",
+      "name": "Goranboy",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-GOY",
+      "name": "Göyçay",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-GYG",
+      "name": "Göygöl",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-HAC",
+      "name": "Hacıqabul",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-IMI",
+      "name": "İmişli",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-ISM",
+      "name": "İsmayıllı",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-KAL",
+      "name": "Kəlbəcər",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-KAN",
+      "name": "Kǝngǝrli",
+      "parent": "AZ-NX",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-KUR",
+      "name": "Kürdəmir",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-LA",
+      "name": "Lənkəran",
+      "type": "Municipality"
+    },
+    {
+      "code": "AZ-LAC",
+      "name": "Laçın",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-LAN",
+      "name": "Lənkəran",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-LER",
+      "name": "Lerik",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-MAS",
+      "name": "Masallı",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-MI",
+      "name": "Mingəçevir",
+      "type": "Municipality"
+    },
+    {
+      "code": "AZ-NA",
+      "name": "Naftalan",
+      "type": "Municipality"
+    },
+    {
+      "code": "AZ-NEF",
+      "name": "Neftçala",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-NV",
+      "name": "Naxçıvan",
+      "parent": "AZ-NX",
+      "type": "Municipality"
+    },
+    {
+      "code": "AZ-NX",
+      "name": "Naxçıvan",
+      "type": "Autonomous republic"
+    },
+    {
+      "code": "AZ-OGU",
+      "name": "Oğuz",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-ORD",
+      "name": "Ordubad",
+      "parent": "AZ-NX",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-QAB",
+      "name": "Qəbələ",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-QAX",
+      "name": "Qax",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-QAZ",
+      "name": "Qazax",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-QBA",
+      "name": "Quba",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-QBI",
+      "name": "Qubadlı",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-QOB",
+      "name": "Qobustan",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-QUS",
+      "name": "Qusar",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-SA",
+      "name": "Şəki",
+      "type": "Municipality"
+    },
+    {
+      "code": "AZ-SAB",
+      "name": "Sabirabad",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-SAD",
+      "name": "Sədərək",
+      "parent": "AZ-NX",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-SAH",
+      "name": "Şahbuz",
+      "parent": "AZ-NX",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-SAK",
+      "name": "Şəki",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-SAL",
+      "name": "Salyan",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-SAR",
+      "name": "Şərur",
+      "parent": "AZ-NX",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-SAT",
+      "name": "Saatlı",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-SBN",
+      "name": "Şabran",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-SIY",
+      "name": "Siyəzən",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-SKR",
+      "name": "Şəmkir",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-SM",
+      "name": "Sumqayıt",
+      "type": "Municipality"
+    },
+    {
+      "code": "AZ-SMI",
+      "name": "Şamaxı",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-SMX",
+      "name": "Samux",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-SR",
+      "name": "Şirvan",
+      "type": "Municipality"
+    },
+    {
+      "code": "AZ-SUS",
+      "name": "Şuşa",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-TAR",
+      "name": "Tərtər",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-TOV",
+      "name": "Tovuz",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-UCA",
+      "name": "Ucar",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-XA",
+      "name": "Xankəndi",
+      "type": "Municipality"
+    },
+    {
+      "code": "AZ-XAC",
+      "name": "Xaçmaz",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-XCI",
+      "name": "Xocalı",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-XIZ",
+      "name": "Xızı",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-XVD",
+      "name": "Xocavənd",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-YAR",
+      "name": "Yardımlı",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-YE",
+      "name": "Yevlax",
+      "type": "Municipality"
+    },
+    {
+      "code": "AZ-YEV",
+      "name": "Yevlax",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-ZAN",
+      "name": "Zəngilan",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-ZAQ",
+      "name": "Zaqatala",
+      "type": "Rayon"
+    },
+    {
+      "code": "AZ-ZAR",
+      "name": "Zərdab",
+      "type": "Rayon"
+    },
+    {
+      "code": "BA-BIH",
+      "name": "Federacija Bosne i Hercegovine",
+      "type": "Entity"
+    },
+    {
+      "code": "BA-BRC",
+      "name": "Brčko distrikt",
+      "type": "District with special status"
+    },
+    {
+      "code": "BA-SRP",
+      "name": "Republika Srpska",
+      "type": "Entity"
+    },
+    {
+      "code": "BB-01",
+      "name": "Christ Church",
+      "type": "Parish"
+    },
+    {
+      "code": "BB-02",
+      "name": "Saint Andrew",
+      "type": "Parish"
+    },
+    {
+      "code": "BB-03",
+      "name": "Saint George",
+      "type": "Parish"
+    },
+    {
+      "code": "BB-04",
+      "name": "Saint James",
+      "type": "Parish"
+    },
+    {
+      "code": "BB-05",
+      "name": "Saint John",
+      "type": "Parish"
+    },
+    {
+      "code": "BB-06",
+      "name": "Saint Joseph",
+      "type": "Parish"
+    },
+    {
+      "code": "BB-07",
+      "name": "Saint Lucy",
+      "type": "Parish"
+    },
+    {
+      "code": "BB-08",
+      "name": "Saint Michael",
+      "type": "Parish"
+    },
+    {
+      "code": "BB-09",
+      "name": "Saint Peter",
+      "type": "Parish"
+    },
+    {
+      "code": "BB-10",
+      "name": "Saint Philip",
+      "type": "Parish"
+    },
+    {
+      "code": "BB-11",
+      "name": "Saint Thomas",
+      "type": "Parish"
+    },
+    {
+      "code": "BD-01",
+      "name": "Bandarban",
+      "parent": "BD-B",
+      "type": "District"
+    },
+    {
+      "code": "BD-02",
+      "name": "Barguna",
+      "parent": "BD-A",
+      "type": "District"
+    },
+    {
+      "code": "BD-03",
+      "name": "Bogura",
+      "parent": "BD-E",
+      "type": "District"
+    },
+    {
+      "code": "BD-04",
+      "name": "Brahmanbaria",
+      "parent": "BD-B",
+      "type": "District"
+    },
+    {
+      "code": "BD-05",
+      "name": "Bagerhat",
+      "parent": "BD-D",
+      "type": "District"
+    },
+    {
+      "code": "BD-06",
+      "name": "Barishal",
+      "parent": "BD-A",
+      "type": "District"
+    },
+    {
+      "code": "BD-07",
+      "name": "Bhola",
+      "parent": "BD-A",
+      "type": "District"
+    },
+    {
+      "code": "BD-08",
+      "name": "Cumilla",
+      "parent": "BD-B",
+      "type": "District"
+    },
+    {
+      "code": "BD-09",
+      "name": "Chandpur",
+      "parent": "BD-B",
+      "type": "District"
+    },
+    {
+      "code": "BD-10",
+      "name": "Chattogram",
+      "parent": "BD-B",
+      "type": "District"
+    },
+    {
+      "code": "BD-11",
+      "name": "Cox's Bazar",
+      "parent": "BD-B",
+      "type": "District"
+    },
+    {
+      "code": "BD-12",
+      "name": "Chuadanga",
+      "parent": "BD-D",
+      "type": "District"
+    },
+    {
+      "code": "BD-13",
+      "name": "Dhaka",
+      "parent": "BD-C",
+      "type": "District"
+    },
+    {
+      "code": "BD-14",
+      "name": "Dinajpur",
+      "parent": "BD-F",
+      "type": "District"
+    },
+    {
+      "code": "BD-15",
+      "name": "Faridpur",
+      "parent": "BD-C",
+      "type": "District"
+    },
+    {
+      "code": "BD-16",
+      "name": "Feni",
+      "parent": "BD-B",
+      "type": "District"
+    },
+    {
+      "code": "BD-17",
+      "name": "Gopalganj",
+      "parent": "BD-C",
+      "type": "District"
+    },
+    {
+      "code": "BD-18",
+      "name": "Gazipur",
+      "parent": "BD-C",
+      "type": "District"
+    },
+    {
+      "code": "BD-19",
+      "name": "Gaibandha",
+      "parent": "BD-F",
+      "type": "District"
+    },
+    {
+      "code": "BD-20",
+      "name": "Habiganj",
+      "parent": "BD-G",
+      "type": "District"
+    },
+    {
+      "code": "BD-21",
+      "name": "Jamalpur",
+      "parent": "BD-H",
+      "type": "District"
+    },
+    {
+      "code": "BD-22",
+      "name": "Jashore",
+      "parent": "BD-D",
+      "type": "District"
+    },
+    {
+      "code": "BD-23",
+      "name": "Jhenaidah",
+      "parent": "BD-D",
+      "type": "District"
+    },
+    {
+      "code": "BD-24",
+      "name": "Joypurhat",
+      "parent": "BD-E",
+      "type": "District"
+    },
+    {
+      "code": "BD-25",
+      "name": "Jhalakathi",
+      "parent": "BD-A",
+      "type": "District"
+    },
+    {
+      "code": "BD-26",
+      "name": "Kishoreganj",
+      "parent": "BD-C",
+      "type": "District"
+    },
+    {
+      "code": "BD-27",
+      "name": "Khulna",
+      "parent": "BD-D",
+      "type": "District"
+    },
+    {
+      "code": "BD-28",
+      "name": "Kurigram",
+      "parent": "BD-F",
+      "type": "District"
+    },
+    {
+      "code": "BD-29",
+      "name": "Khagrachhari",
+      "parent": "BD-B",
+      "type": "District"
+    },
+    {
+      "code": "BD-30",
+      "name": "Kushtia",
+      "parent": "BD-D",
+      "type": "District"
+    },
+    {
+      "code": "BD-31",
+      "name": "Lakshmipur",
+      "parent": "BD-B",
+      "type": "District"
+    },
+    {
+      "code": "BD-32",
+      "name": "Lalmonirhat",
+      "parent": "BD-F",
+      "type": "District"
+    },
+    {
+      "code": "BD-33",
+      "name": "Manikganj",
+      "parent": "BD-C",
+      "type": "District"
+    },
+    {
+      "code": "BD-34",
+      "name": "Mymensingh",
+      "parent": "BD-H",
+      "type": "District"
+    },
+    {
+      "code": "BD-35",
+      "name": "Munshiganj",
+      "parent": "BD-C",
+      "type": "District"
+    },
+    {
+      "code": "BD-36",
+      "name": "Madaripur",
+      "parent": "BD-C",
+      "type": "District"
+    },
+    {
+      "code": "BD-37",
+      "name": "Magura",
+      "parent": "BD-D",
+      "type": "District"
+    },
+    {
+      "code": "BD-38",
+      "name": "Moulvibazar",
+      "parent": "BD-G",
+      "type": "District"
+    },
+    {
+      "code": "BD-39",
+      "name": "Meherpur",
+      "parent": "BD-D",
+      "type": "District"
+    },
+    {
+      "code": "BD-40",
+      "name": "Narayanganj",
+      "parent": "BD-C",
+      "type": "District"
+    },
+    {
+      "code": "BD-41",
+      "name": "Netrakona",
+      "parent": "BD-H",
+      "type": "District"
+    },
+    {
+      "code": "BD-42",
+      "name": "Narsingdi",
+      "parent": "BD-C",
+      "type": "District"
+    },
+    {
+      "code": "BD-43",
+      "name": "Narail",
+      "parent": "BD-D",
+      "type": "District"
+    },
+    {
+      "code": "BD-44",
+      "name": "Natore",
+      "parent": "BD-E",
+      "type": "District"
+    },
+    {
+      "code": "BD-45",
+      "name": "Chapai Nawabganj",
+      "parent": "BD-E",
+      "type": "District"
+    },
+    {
+      "code": "BD-46",
+      "name": "Nilphamari",
+      "parent": "BD-F",
+      "type": "District"
+    },
+    {
+      "code": "BD-47",
+      "name": "Noakhali",
+      "parent": "BD-B",
+      "type": "District"
+    },
+    {
+      "code": "BD-48",
+      "name": "Naogaon",
+      "parent": "BD-E",
+      "type": "District"
+    },
+    {
+      "code": "BD-49",
+      "name": "Pabna",
+      "parent": "BD-E",
+      "type": "District"
+    },
+    {
+      "code": "BD-50",
+      "name": "Pirojpur",
+      "parent": "BD-A",
+      "type": "District"
+    },
+    {
+      "code": "BD-51",
+      "name": "Patuakhali",
+      "parent": "BD-A",
+      "type": "District"
+    },
+    {
+      "code": "BD-52",
+      "name": "Panchagarh",
+      "parent": "BD-F",
+      "type": "District"
+    },
+    {
+      "code": "BD-53",
+      "name": "Rajbari",
+      "parent": "BD-C",
+      "type": "District"
+    },
+    {
+      "code": "BD-54",
+      "name": "Rajshahi",
+      "parent": "BD-E",
+      "type": "District"
+    },
+    {
+      "code": "BD-55",
+      "name": "Rangpur",
+      "parent": "BD-F",
+      "type": "District"
+    },
+    {
+      "code": "BD-56",
+      "name": "Rangamati",
+      "parent": "BD-B",
+      "type": "District"
+    },
+    {
+      "code": "BD-57",
+      "name": "Sherpur",
+      "parent": "BD-H",
+      "type": "District"
+    },
+    {
+      "code": "BD-58",
+      "name": "Satkhira",
+      "parent": "BD-D",
+      "type": "District"
+    },
+    {
+      "code": "BD-59",
+      "name": "Sirajganj",
+      "parent": "BD-E",
+      "type": "District"
+    },
+    {
+      "code": "BD-60",
+      "name": "Sylhet",
+      "parent": "BD-G",
+      "type": "District"
+    },
+    {
+      "code": "BD-61",
+      "name": "Sunamganj",
+      "parent": "BD-G",
+      "type": "District"
+    },
+    {
+      "code": "BD-62",
+      "name": "Shariatpur",
+      "parent": "BD-C",
+      "type": "District"
+    },
+    {
+      "code": "BD-63",
+      "name": "Tangail",
+      "parent": "BD-C",
+      "type": "District"
+    },
+    {
+      "code": "BD-64",
+      "name": "Thakurgaon",
+      "parent": "BD-F",
+      "type": "District"
+    },
+    {
+      "code": "BD-A",
+      "name": "Barishal",
+      "type": "Division"
+    },
+    {
+      "code": "BD-B",
+      "name": "Chattogram",
+      "type": "Division"
+    },
+    {
+      "code": "BD-C",
+      "name": "Dhaka",
+      "type": "Division"
+    },
+    {
+      "code": "BD-D",
+      "name": "Khulna",
+      "type": "Division"
+    },
+    {
+      "code": "BD-E",
+      "name": "Rajshahi",
+      "type": "Division"
+    },
+    {
+      "code": "BD-F",
+      "name": "Rangpur",
+      "type": "Division"
+    },
+    {
+      "code": "BD-G",
+      "name": "Sylhet",
+      "type": "Division"
+    },
+    {
+      "code": "BD-H",
+      "name": "Mymensingh",
+      "type": "Division"
+    },
+    {
+      "code": "BE-BRU",
+      "name": "Bruxelles-Capitale, Région de",
+      "type": "Region"
+    },
+    {
+      "code": "BE-VAN",
+      "name": "Antwerpen",
+      "parent": "BE-VLG",
+      "type": "Province"
+    },
+    {
+      "code": "BE-VBR",
+      "name": "Vlaams-Brabant",
+      "parent": "BE-VLG",
+      "type": "Province"
+    },
+    {
+      "code": "BE-VLG",
+      "name": "Vlaams Gewest",
+      "type": "Region"
+    },
+    {
+      "code": "BE-VLI",
+      "name": "Limburg",
+      "parent": "BE-VLG",
+      "type": "Province"
+    },
+    {
+      "code": "BE-VOV",
+      "name": "Oost-Vlaanderen",
+      "parent": "BE-VLG",
+      "type": "Province"
+    },
+    {
+      "code": "BE-VWV",
+      "name": "West-Vlaanderen",
+      "parent": "BE-VLG",
+      "type": "Province"
+    },
+    {
+      "code": "BE-WAL",
+      "name": "wallonne, Région",
+      "type": "Region"
+    },
+    {
+      "code": "BE-WBR",
+      "name": "Brabant wallon",
+      "parent": "BE-WAL",
+      "type": "Province"
+    },
+    {
+      "code": "BE-WHT",
+      "name": "Hainaut",
+      "parent": "BE-WAL",
+      "type": "Province"
+    },
+    {
+      "code": "BE-WLG",
+      "name": "Liège",
+      "parent": "BE-WAL",
+      "type": "Province"
+    },
+    {
+      "code": "BE-WLX",
+      "name": "Luxembourg",
+      "parent": "BE-WAL",
+      "type": "Province"
+    },
+    {
+      "code": "BE-WNA",
+      "name": "Namur",
+      "parent": "BE-WAL",
+      "type": "Province"
+    },
+    {
+      "code": "BF-01",
+      "name": "Boucle du Mouhoun",
+      "type": "Region"
+    },
+    {
+      "code": "BF-02",
+      "name": "Cascades",
+      "type": "Region"
+    },
+    {
+      "code": "BF-03",
+      "name": "Centre",
+      "type": "Region"
+    },
+    {
+      "code": "BF-04",
+      "name": "Centre-Est",
+      "type": "Region"
+    },
+    {
+      "code": "BF-05",
+      "name": "Centre-Nord",
+      "type": "Region"
+    },
+    {
+      "code": "BF-06",
+      "name": "Centre-Ouest",
+      "type": "Region"
+    },
+    {
+      "code": "BF-07",
+      "name": "Centre-Sud",
+      "type": "Region"
+    },
+    {
+      "code": "BF-08",
+      "name": "Est",
+      "type": "Region"
+    },
+    {
+      "code": "BF-09",
+      "name": "Hauts-Bassins",
+      "type": "Region"
+    },
+    {
+      "code": "BF-10",
+      "name": "Nord",
+      "type": "Region"
+    },
+    {
+      "code": "BF-11",
+      "name": "Plateau-Central",
+      "type": "Region"
+    },
+    {
+      "code": "BF-12",
+      "name": "Sahel",
+      "type": "Region"
+    },
+    {
+      "code": "BF-13",
+      "name": "Sud-Ouest",
+      "type": "Region"
+    },
+    {
+      "code": "BF-BAL",
+      "name": "Balé",
+      "parent": "BF-01",
+      "type": "Province"
+    },
+    {
+      "code": "BF-BAM",
+      "name": "Bam",
+      "parent": "BF-05",
+      "type": "Province"
+    },
+    {
+      "code": "BF-BAN",
+      "name": "Banwa",
+      "parent": "BF-01",
+      "type": "Province"
+    },
+    {
+      "code": "BF-BAZ",
+      "name": "Bazèga",
+      "parent": "BF-07",
+      "type": "Province"
+    },
+    {
+      "code": "BF-BGR",
+      "name": "Bougouriba",
+      "parent": "BF-13",
+      "type": "Province"
+    },
+    {
+      "code": "BF-BLG",
+      "name": "Boulgou",
+      "parent": "BF-04",
+      "type": "Province"
+    },
+    {
+      "code": "BF-BLK",
+      "name": "Boulkiemdé",
+      "parent": "BF-06",
+      "type": "Province"
+    },
+    {
+      "code": "BF-COM",
+      "name": "Comoé",
+      "parent": "BF-02",
+      "type": "Province"
+    },
+    {
+      "code": "BF-GAN",
+      "name": "Ganzourgou",
+      "parent": "BF-11",
+      "type": "Province"
+    },
+    {
+      "code": "BF-GNA",
+      "name": "Gnagna",
+      "parent": "BF-08",
+      "type": "Province"
+    },
+    {
+      "code": "BF-GOU",
+      "name": "Gourma",
+      "parent": "BF-08",
+      "type": "Province"
+    },
+    {
+      "code": "BF-HOU",
+      "name": "Houet",
+      "parent": "BF-09",
+      "type": "Province"
+    },
+    {
+      "code": "BF-IOB",
+      "name": "Ioba",
+      "parent": "BF-13",
+      "type": "Province"
+    },
+    {
+      "code": "BF-KAD",
+      "name": "Kadiogo",
+      "parent": "BF-03",
+      "type": "Province"
+    },
+    {
+      "code": "BF-KEN",
+      "name": "Kénédougou",
+      "parent": "BF-09",
+      "type": "Province"
+    },
+    {
+      "code": "BF-KMD",
+      "name": "Komondjari",
+      "parent": "BF-08",
+      "type": "Province"
+    },
+    {
+      "code": "BF-KMP",
+      "name": "Kompienga",
+      "parent": "BF-08",
+      "type": "Province"
+    },
+    {
+      "code": "BF-KOP",
+      "name": "Koulpélogo",
+      "parent": "BF-04",
+      "type": "Province"
+    },
+    {
+      "code": "BF-KOS",
+      "name": "Kossi",
+      "parent": "BF-01",
+      "type": "Province"
+    },
+    {
+      "code": "BF-KOT",
+      "name": "Kouritenga",
+      "parent": "BF-04",
+      "type": "Province"
+    },
+    {
+      "code": "BF-KOW",
+      "name": "Kourwéogo",
+      "parent": "BF-11",
+      "type": "Province"
+    },
+    {
+      "code": "BF-LER",
+      "name": "Léraba",
+      "parent": "BF-02",
+      "type": "Province"
+    },
+    {
+      "code": "BF-LOR",
+      "name": "Loroum",
+      "parent": "BF-10",
+      "type": "Province"
+    },
+    {
+      "code": "BF-MOU",
+      "name": "Mouhoun",
+      "parent": "BF-01",
+      "type": "Province"
+    },
+    {
+      "code": "BF-NAM",
+      "name": "Namentenga",
+      "parent": "BF-05",
+      "type": "Province"
+    },
+    {
+      "code": "BF-NAO",
+      "name": "Nahouri",
+      "parent": "BF-07",
+      "type": "Province"
+    },
+    {
+      "code": "BF-NAY",
+      "name": "Nayala",
+      "parent": "BF-01",
+      "type": "Province"
+    },
+    {
+      "code": "BF-NOU",
+      "name": "Noumbiel",
+      "parent": "BF-13",
+      "type": "Province"
+    },
+    {
+      "code": "BF-OUB",
+      "name": "Oubritenga",
+      "parent": "BF-11",
+      "type": "Province"
+    },
+    {
+      "code": "BF-OUD",
+      "name": "Oudalan",
+      "parent": "BF-12",
+      "type": "Province"
+    },
+    {
+      "code": "BF-PAS",
+      "name": "Passoré",
+      "parent": "BF-10",
+      "type": "Province"
+    },
+    {
+      "code": "BF-PON",
+      "name": "Poni",
+      "parent": "BF-13",
+      "type": "Province"
+    },
+    {
+      "code": "BF-SEN",
+      "name": "Séno",
+      "parent": "BF-12",
+      "type": "Province"
+    },
+    {
+      "code": "BF-SIS",
+      "name": "Sissili",
+      "parent": "BF-06",
+      "type": "Province"
+    },
+    {
+      "code": "BF-SMT",
+      "name": "Sanmatenga",
+      "parent": "BF-05",
+      "type": "Province"
+    },
+    {
+      "code": "BF-SNG",
+      "name": "Sanguié",
+      "parent": "BF-06",
+      "type": "Province"
+    },
+    {
+      "code": "BF-SOM",
+      "name": "Soum",
+      "parent": "BF-12",
+      "type": "Province"
+    },
+    {
+      "code": "BF-SOR",
+      "name": "Sourou",
+      "parent": "BF-01",
+      "type": "Province"
+    },
+    {
+      "code": "BF-TAP",
+      "name": "Tapoa",
+      "parent": "BF-08",
+      "type": "Province"
+    },
+    {
+      "code": "BF-TUI",
+      "name": "Tuy",
+      "parent": "BF-09",
+      "type": "Province"
+    },
+    {
+      "code": "BF-YAG",
+      "name": "Yagha",
+      "parent": "BF-12",
+      "type": "Province"
+    },
+    {
+      "code": "BF-YAT",
+      "name": "Yatenga",
+      "parent": "BF-10",
+      "type": "Province"
+    },
+    {
+      "code": "BF-ZIR",
+      "name": "Ziro",
+      "parent": "BF-06",
+      "type": "Province"
+    },
+    {
+      "code": "BF-ZON",
+      "name": "Zondoma",
+      "parent": "BF-10",
+      "type": "Province"
+    },
+    {
+      "code": "BF-ZOU",
+      "name": "Zoundwéogo",
+      "parent": "BF-07",
+      "type": "Province"
+    },
+    {
+      "code": "BG-01",
+      "name": "Blagoevgrad",
+      "type": "District"
+    },
+    {
+      "code": "BG-02",
+      "name": "Burgas",
+      "type": "District"
+    },
+    {
+      "code": "BG-03",
+      "name": "Varna",
+      "type": "District"
+    },
+    {
+      "code": "BG-04",
+      "name": "Veliko Tarnovo",
+      "type": "District"
+    },
+    {
+      "code": "BG-05",
+      "name": "Vidin",
+      "type": "District"
+    },
+    {
+      "code": "BG-06",
+      "name": "Vratsa",
+      "type": "District"
+    },
+    {
+      "code": "BG-07",
+      "name": "Gabrovo",
+      "type": "District"
+    },
+    {
+      "code": "BG-08",
+      "name": "Dobrich",
+      "type": "District"
+    },
+    {
+      "code": "BG-09",
+      "name": "Kardzhali",
+      "type": "District"
+    },
+    {
+      "code": "BG-10",
+      "name": "Kyustendil",
+      "type": "District"
+    },
+    {
+      "code": "BG-11",
+      "name": "Lovech",
+      "type": "District"
+    },
+    {
+      "code": "BG-12",
+      "name": "Montana",
+      "type": "District"
+    },
+    {
+      "code": "BG-13",
+      "name": "Pazardzhik",
+      "type": "District"
+    },
+    {
+      "code": "BG-14",
+      "name": "Pernik",
+      "type": "District"
+    },
+    {
+      "code": "BG-15",
+      "name": "Pleven",
+      "type": "District"
+    },
+    {
+      "code": "BG-16",
+      "name": "Plovdiv",
+      "type": "District"
+    },
+    {
+      "code": "BG-17",
+      "name": "Razgrad",
+      "type": "District"
+    },
+    {
+      "code": "BG-18",
+      "name": "Ruse",
+      "type": "District"
+    },
+    {
+      "code": "BG-19",
+      "name": "Silistra",
+      "type": "District"
+    },
+    {
+      "code": "BG-20",
+      "name": "Sliven",
+      "type": "District"
+    },
+    {
+      "code": "BG-21",
+      "name": "Smolyan",
+      "type": "District"
+    },
+    {
+      "code": "BG-22",
+      "name": "Sofia (stolitsa)",
+      "type": "District"
+    },
+    {
+      "code": "BG-23",
+      "name": "Sofia",
+      "type": "District"
+    },
+    {
+      "code": "BG-24",
+      "name": "Stara Zagora",
+      "type": "District"
+    },
+    {
+      "code": "BG-25",
+      "name": "Targovishte",
+      "type": "District"
+    },
+    {
+      "code": "BG-26",
+      "name": "Haskovo",
+      "type": "District"
+    },
+    {
+      "code": "BG-27",
+      "name": "Shumen",
+      "type": "District"
+    },
+    {
+      "code": "BG-28",
+      "name": "Yambol",
+      "type": "District"
+    },
+    {
+      "code": "BH-13",
+      "name": "Al ‘Āşimah",
+      "type": "Governorate"
+    },
+    {
+      "code": "BH-14",
+      "name": "Al Janūbīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "BH-15",
+      "name": "Al Muḩarraq",
+      "type": "Governorate"
+    },
+    {
+      "code": "BH-17",
+      "name": "Ash Shamālīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "BI-BB",
+      "name": "Bubanza",
+      "type": "Province"
+    },
+    {
+      "code": "BI-BL",
+      "name": "Bujumbura Rural",
+      "type": "Province"
+    },
+    {
+      "code": "BI-BM",
+      "name": "Bujumbura Mairie",
+      "type": "Province"
+    },
+    {
+      "code": "BI-BR",
+      "name": "Bururi",
+      "type": "Province"
+    },
+    {
+      "code": "BI-CA",
+      "name": "Cankuzo",
+      "type": "Province"
+    },
+    {
+      "code": "BI-CI",
+      "name": "Cibitoke",
+      "type": "Province"
+    },
+    {
+      "code": "BI-GI",
+      "name": "Gitega",
+      "type": "Province"
+    },
+    {
+      "code": "BI-KI",
+      "name": "Kirundo",
+      "type": "Province"
+    },
+    {
+      "code": "BI-KR",
+      "name": "Karuzi",
+      "type": "Province"
+    },
+    {
+      "code": "BI-KY",
+      "name": "Kayanza",
+      "type": "Province"
+    },
+    {
+      "code": "BI-MA",
+      "name": "Makamba",
+      "type": "Province"
+    },
+    {
+      "code": "BI-MU",
+      "name": "Muramvya",
+      "type": "Province"
+    },
+    {
+      "code": "BI-MW",
+      "name": "Mwaro",
+      "type": "Province"
+    },
+    {
+      "code": "BI-MY",
+      "name": "Muyinga",
+      "type": "Province"
+    },
+    {
+      "code": "BI-NG",
+      "name": "Ngozi",
+      "type": "Province"
+    },
+    {
+      "code": "BI-RM",
+      "name": "Rumonge",
+      "type": "Province"
+    },
+    {
+      "code": "BI-RT",
+      "name": "Rutana",
+      "type": "Province"
+    },
+    {
+      "code": "BI-RY",
+      "name": "Ruyigi",
+      "type": "Province"
+    },
+    {
+      "code": "BJ-AK",
+      "name": "Atacora",
+      "type": "Department"
+    },
+    {
+      "code": "BJ-AL",
+      "name": "Alibori",
+      "type": "Department"
+    },
+    {
+      "code": "BJ-AQ",
+      "name": "Atlantique",
+      "type": "Department"
+    },
+    {
+      "code": "BJ-BO",
+      "name": "Borgou",
+      "type": "Department"
+    },
+    {
+      "code": "BJ-CO",
+      "name": "Collines",
+      "type": "Department"
+    },
+    {
+      "code": "BJ-DO",
+      "name": "Donga",
+      "type": "Department"
+    },
+    {
+      "code": "BJ-KO",
+      "name": "Couffo",
+      "type": "Department"
+    },
+    {
+      "code": "BJ-LI",
+      "name": "Littoral",
+      "type": "Department"
+    },
+    {
+      "code": "BJ-MO",
+      "name": "Mono",
+      "type": "Department"
+    },
+    {
+      "code": "BJ-OU",
+      "name": "Ouémé",
+      "type": "Department"
+    },
+    {
+      "code": "BJ-PL",
+      "name": "Plateau",
+      "type": "Department"
+    },
+    {
+      "code": "BJ-ZO",
+      "name": "Zou",
+      "type": "Department"
+    },
+    {
+      "code": "BN-BE",
+      "name": "Belait",
+      "type": "District"
+    },
+    {
+      "code": "BN-BM",
+      "name": "Brunei-Muara",
+      "type": "District"
+    },
+    {
+      "code": "BN-TE",
+      "name": "Temburong",
+      "type": "District"
+    },
+    {
+      "code": "BN-TU",
+      "name": "Tutong",
+      "type": "District"
+    },
+    {
+      "code": "BO-B",
+      "name": "El Beni",
+      "type": "Department"
+    },
+    {
+      "code": "BO-C",
+      "name": "Cochabamba",
+      "type": "Department"
+    },
+    {
+      "code": "BO-H",
+      "name": "Chuquisaca",
+      "type": "Department"
+    },
+    {
+      "code": "BO-L",
+      "name": "La Paz",
+      "type": "Department"
+    },
+    {
+      "code": "BO-N",
+      "name": "Pando",
+      "type": "Department"
+    },
+    {
+      "code": "BO-O",
+      "name": "Oruro",
+      "type": "Department"
+    },
+    {
+      "code": "BO-P",
+      "name": "Potosí",
+      "type": "Department"
+    },
+    {
+      "code": "BO-S",
+      "name": "Santa Cruz",
+      "type": "Department"
+    },
+    {
+      "code": "BO-T",
+      "name": "Tarija",
+      "type": "Department"
+    },
+    {
+      "code": "BQ-BO",
+      "name": "Bonaire",
+      "type": "Special municipality"
+    },
+    {
+      "code": "BQ-SA",
+      "name": "Saba",
+      "type": "Special municipality"
+    },
+    {
+      "code": "BQ-SE",
+      "name": "Sint Eustatius",
+      "type": "Special municipality"
+    },
+    {
+      "code": "BR-AC",
+      "name": "Acre",
+      "type": "State"
+    },
+    {
+      "code": "BR-AL",
+      "name": "Alagoas",
+      "type": "State"
+    },
+    {
+      "code": "BR-AM",
+      "name": "Amazonas",
+      "type": "State"
+    },
+    {
+      "code": "BR-AP",
+      "name": "Amapá",
+      "type": "State"
+    },
+    {
+      "code": "BR-BA",
+      "name": "Bahia",
+      "type": "State"
+    },
+    {
+      "code": "BR-CE",
+      "name": "Ceará",
+      "type": "State"
+    },
+    {
+      "code": "BR-DF",
+      "name": "Distrito Federal",
+      "type": "Federal district"
+    },
+    {
+      "code": "BR-ES",
+      "name": "Espírito Santo",
+      "type": "State"
+    },
+    {
+      "code": "BR-GO",
+      "name": "Goiás",
+      "type": "State"
+    },
+    {
+      "code": "BR-MA",
+      "name": "Maranhão",
+      "type": "State"
+    },
+    {
+      "code": "BR-MG",
+      "name": "Minas Gerais",
+      "type": "State"
+    },
+    {
+      "code": "BR-MS",
+      "name": "Mato Grosso do Sul",
+      "type": "State"
+    },
+    {
+      "code": "BR-MT",
+      "name": "Mato Grosso",
+      "type": "State"
+    },
+    {
+      "code": "BR-PA",
+      "name": "Pará",
+      "type": "State"
+    },
+    {
+      "code": "BR-PB",
+      "name": "Paraíba",
+      "type": "State"
+    },
+    {
+      "code": "BR-PE",
+      "name": "Pernambuco",
+      "type": "State"
+    },
+    {
+      "code": "BR-PI",
+      "name": "Piauí",
+      "type": "State"
+    },
+    {
+      "code": "BR-PR",
+      "name": "Paraná",
+      "type": "State"
+    },
+    {
+      "code": "BR-RJ",
+      "name": "Rio de Janeiro",
+      "type": "State"
+    },
+    {
+      "code": "BR-RN",
+      "name": "Rio Grande do Norte",
+      "type": "State"
+    },
+    {
+      "code": "BR-RO",
+      "name": "Rondônia",
+      "type": "State"
+    },
+    {
+      "code": "BR-RR",
+      "name": "Roraima",
+      "type": "State"
+    },
+    {
+      "code": "BR-RS",
+      "name": "Rio Grande do Sul",
+      "type": "State"
+    },
+    {
+      "code": "BR-SC",
+      "name": "Santa Catarina",
+      "type": "State"
+    },
+    {
+      "code": "BR-SE",
+      "name": "Sergipe",
+      "type": "State"
+    },
+    {
+      "code": "BR-SP",
+      "name": "São Paulo",
+      "type": "State"
+    },
+    {
+      "code": "BR-TO",
+      "name": "Tocantins",
+      "type": "State"
+    },
+    {
+      "code": "BS-AK",
+      "name": "Acklins",
+      "type": "District"
+    },
+    {
+      "code": "BS-BI",
+      "name": "Bimini",
+      "type": "District"
+    },
+    {
+      "code": "BS-BP",
+      "name": "Black Point",
+      "type": "District"
+    },
+    {
+      "code": "BS-BY",
+      "name": "Berry Islands",
+      "type": "District"
+    },
+    {
+      "code": "BS-CE",
+      "name": "Central Eleuthera",
+      "type": "District"
+    },
+    {
+      "code": "BS-CI",
+      "name": "Cat Island",
+      "type": "District"
+    },
+    {
+      "code": "BS-CK",
+      "name": "Crooked Island and Long Cay",
+      "type": "District"
+    },
+    {
+      "code": "BS-CO",
+      "name": "Central Abaco",
+      "type": "District"
+    },
+    {
+      "code": "BS-CS",
+      "name": "Central Andros",
+      "type": "District"
+    },
+    {
+      "code": "BS-EG",
+      "name": "East Grand Bahama",
+      "type": "District"
+    },
+    {
+      "code": "BS-EX",
+      "name": "Exuma",
+      "type": "District"
+    },
+    {
+      "code": "BS-FP",
+      "name": "City of Freeport",
+      "type": "District"
+    },
+    {
+      "code": "BS-GC",
+      "name": "Grand Cay",
+      "type": "District"
+    },
+    {
+      "code": "BS-HI",
+      "name": "Harbour Island",
+      "type": "District"
+    },
+    {
+      "code": "BS-HT",
+      "name": "Hope Town",
+      "type": "District"
+    },
+    {
+      "code": "BS-IN",
+      "name": "Inagua",
+      "type": "District"
+    },
+    {
+      "code": "BS-LI",
+      "name": "Long Island",
+      "type": "District"
+    },
+    {
+      "code": "BS-MC",
+      "name": "Mangrove Cay",
+      "type": "District"
+    },
+    {
+      "code": "BS-MG",
+      "name": "Mayaguana",
+      "type": "District"
+    },
+    {
+      "code": "BS-MI",
+      "name": "Moore's Island",
+      "type": "District"
+    },
+    {
+      "code": "BS-NE",
+      "name": "North Eleuthera",
+      "type": "District"
+    },
+    {
+      "code": "BS-NO",
+      "name": "North Abaco",
+      "type": "District"
+    },
+    {
+      "code": "BS-NP",
+      "name": "New Providence",
+      "type": "Island"
+    },
+    {
+      "code": "BS-NS",
+      "name": "North Andros",
+      "type": "District"
+    },
+    {
+      "code": "BS-RC",
+      "name": "Rum Cay",
+      "type": "District"
+    },
+    {
+      "code": "BS-RI",
+      "name": "Ragged Island",
+      "type": "District"
+    },
+    {
+      "code": "BS-SA",
+      "name": "South Andros",
+      "type": "District"
+    },
+    {
+      "code": "BS-SE",
+      "name": "South Eleuthera",
+      "type": "District"
+    },
+    {
+      "code": "BS-SO",
+      "name": "South Abaco",
+      "type": "District"
+    },
+    {
+      "code": "BS-SS",
+      "name": "San Salvador",
+      "type": "District"
+    },
+    {
+      "code": "BS-SW",
+      "name": "Spanish Wells",
+      "type": "District"
+    },
+    {
+      "code": "BS-WG",
+      "name": "West Grand Bahama",
+      "type": "District"
+    },
+    {
+      "code": "BT-11",
+      "name": "Paro",
+      "type": "District"
+    },
+    {
+      "code": "BT-12",
+      "name": "Chhukha",
+      "type": "District"
+    },
+    {
+      "code": "BT-13",
+      "name": "Haa",
+      "type": "District"
+    },
+    {
+      "code": "BT-14",
+      "name": "Samtse",
+      "type": "District"
+    },
+    {
+      "code": "BT-15",
+      "name": "Thimphu",
+      "type": "District"
+    },
+    {
+      "code": "BT-21",
+      "name": "Tsirang",
+      "type": "District"
+    },
+    {
+      "code": "BT-22",
+      "name": "Dagana",
+      "type": "District"
+    },
+    {
+      "code": "BT-23",
+      "name": "Punakha",
+      "type": "District"
+    },
+    {
+      "code": "BT-24",
+      "name": "Wangdue Phodrang",
+      "type": "District"
+    },
+    {
+      "code": "BT-31",
+      "name": "Sarpang",
+      "type": "District"
+    },
+    {
+      "code": "BT-32",
+      "name": "Trongsa",
+      "type": "District"
+    },
+    {
+      "code": "BT-33",
+      "name": "Bumthang",
+      "type": "District"
+    },
+    {
+      "code": "BT-34",
+      "name": "Zhemgang",
+      "type": "District"
+    },
+    {
+      "code": "BT-41",
+      "name": "Trashigang",
+      "type": "District"
+    },
+    {
+      "code": "BT-42",
+      "name": "Monggar",
+      "type": "District"
+    },
+    {
+      "code": "BT-43",
+      "name": "Pema Gatshel",
+      "type": "District"
+    },
+    {
+      "code": "BT-44",
+      "name": "Lhuentse",
+      "type": "District"
+    },
+    {
+      "code": "BT-45",
+      "name": "Samdrup Jongkhar",
+      "type": "District"
+    },
+    {
+      "code": "BT-GA",
+      "name": "Gasa",
+      "type": "District"
+    },
+    {
+      "code": "BT-TY",
+      "name": "Trashi Yangtse",
+      "type": "District"
+    },
+    {
+      "code": "BW-CE",
+      "name": "Central",
+      "type": "District"
+    },
+    {
+      "code": "BW-CH",
+      "name": "Chobe",
+      "type": "District"
+    },
+    {
+      "code": "BW-FR",
+      "name": "Francistown",
+      "type": "City"
+    },
+    {
+      "code": "BW-GA",
+      "name": "Gaborone",
+      "type": "City"
+    },
+    {
+      "code": "BW-GH",
+      "name": "Ghanzi",
+      "type": "District"
+    },
+    {
+      "code": "BW-JW",
+      "name": "Jwaneng",
+      "type": "Town"
+    },
+    {
+      "code": "BW-KG",
+      "name": "Kgalagadi",
+      "type": "District"
+    },
+    {
+      "code": "BW-KL",
+      "name": "Kgatleng",
+      "type": "District"
+    },
+    {
+      "code": "BW-KW",
+      "name": "Kweneng",
+      "type": "District"
+    },
+    {
+      "code": "BW-LO",
+      "name": "Lobatse",
+      "type": "Town"
+    },
+    {
+      "code": "BW-NE",
+      "name": "North East",
+      "type": "District"
+    },
+    {
+      "code": "BW-NW",
+      "name": "North West",
+      "type": "District"
+    },
+    {
+      "code": "BW-SE",
+      "name": "South East",
+      "type": "District"
+    },
+    {
+      "code": "BW-SO",
+      "name": "Southern",
+      "type": "District"
+    },
+    {
+      "code": "BW-SP",
+      "name": "Selibe Phikwe",
+      "type": "Town"
+    },
+    {
+      "code": "BW-ST",
+      "name": "Sowa Town",
+      "type": "Town"
+    },
+    {
+      "code": "BY-BR",
+      "name": "Bresckaja voblasć",
+      "type": "Oblast"
+    },
+    {
+      "code": "BY-HM",
+      "name": "Gorod Minsk",
+      "type": "City"
+    },
+    {
+      "code": "BY-HO",
+      "name": "Gomel'skaja oblast'",
+      "type": "Oblast"
+    },
+    {
+      "code": "BY-HR",
+      "name": "Grodnenskaja oblast'",
+      "type": "Oblast"
+    },
+    {
+      "code": "BY-MA",
+      "name": "Mahilioŭskaja voblasć",
+      "type": "Oblast"
+    },
+    {
+      "code": "BY-MI",
+      "name": "Minskaja oblast'",
+      "type": "Oblast"
+    },
+    {
+      "code": "BY-VI",
+      "name": "Viciebskaja voblasć",
+      "type": "Oblast"
+    },
+    {
+      "code": "BZ-BZ",
+      "name": "Belize",
+      "type": "District"
+    },
+    {
+      "code": "BZ-CY",
+      "name": "Cayo",
+      "type": "District"
+    },
+    {
+      "code": "BZ-CZL",
+      "name": "Corozal",
+      "type": "District"
+    },
+    {
+      "code": "BZ-OW",
+      "name": "Orange Walk",
+      "type": "District"
+    },
+    {
+      "code": "BZ-SC",
+      "name": "Stann Creek",
+      "type": "District"
+    },
+    {
+      "code": "BZ-TOL",
+      "name": "Toledo",
+      "type": "District"
+    },
+    {
+      "code": "CA-AB",
+      "name": "Alberta",
+      "type": "Province"
+    },
+    {
+      "code": "CA-BC",
+      "name": "British Columbia",
+      "type": "Province"
+    },
+    {
+      "code": "CA-MB",
+      "name": "Manitoba",
+      "type": "Province"
+    },
+    {
+      "code": "CA-NB",
+      "name": "New Brunswick",
+      "type": "Province"
+    },
+    {
+      "code": "CA-NL",
+      "name": "Newfoundland and Labrador",
+      "type": "Province"
+    },
+    {
+      "code": "CA-NS",
+      "name": "Nova Scotia",
+      "type": "Province"
+    },
+    {
+      "code": "CA-NT",
+      "name": "Northwest Territories",
+      "type": "Territory"
+    },
+    {
+      "code": "CA-NU",
+      "name": "Nunavut",
+      "type": "Territory"
+    },
+    {
+      "code": "CA-ON",
+      "name": "Ontario",
+      "type": "Province"
+    },
+    {
+      "code": "CA-PE",
+      "name": "Prince Edward Island",
+      "type": "Province"
+    },
+    {
+      "code": "CA-QC",
+      "name": "Quebec",
+      "type": "Province"
+    },
+    {
+      "code": "CA-SK",
+      "name": "Saskatchewan",
+      "type": "Province"
+    },
+    {
+      "code": "CA-YT",
+      "name": "Yukon",
+      "type": "Territory"
+    },
+    {
+      "code": "CD-BC",
+      "name": "Kongo Central",
+      "type": "Province"
+    },
+    {
+      "code": "CD-BU",
+      "name": "Bas-Uélé",
+      "type": "Province"
+    },
+    {
+      "code": "CD-EQ",
+      "name": "Équateur",
+      "type": "Province"
+    },
+    {
+      "code": "CD-HK",
+      "name": "Haut-Katanga",
+      "type": "Province"
+    },
+    {
+      "code": "CD-HL",
+      "name": "Haut-Lomami",
+      "type": "Province"
+    },
+    {
+      "code": "CD-HU",
+      "name": "Haut-Uélé",
+      "type": "Province"
+    },
+    {
+      "code": "CD-IT",
+      "name": "Ituri",
+      "type": "Province"
+    },
+    {
+      "code": "CD-KC",
+      "name": "Kasaï Central",
+      "type": "Province"
+    },
+    {
+      "code": "CD-KE",
+      "name": "Kasaï Oriental",
+      "type": "Province"
+    },
+    {
+      "code": "CD-KG",
+      "name": "Kwango",
+      "type": "Province"
+    },
+    {
+      "code": "CD-KL",
+      "name": "Kwilu",
+      "type": "Province"
+    },
+    {
+      "code": "CD-KN",
+      "name": "Kinshasa",
+      "type": "City"
+    },
+    {
+      "code": "CD-KS",
+      "name": "Kasaï",
+      "type": "Province"
+    },
+    {
+      "code": "CD-LO",
+      "name": "Lomami",
+      "type": "Province"
+    },
+    {
+      "code": "CD-LU",
+      "name": "Lualaba",
+      "type": "Province"
+    },
+    {
+      "code": "CD-MA",
+      "name": "Maniema",
+      "type": "Province"
+    },
+    {
+      "code": "CD-MN",
+      "name": "Mai-Ndombe",
+      "type": "Province"
+    },
+    {
+      "code": "CD-MO",
+      "name": "Mongala",
+      "type": "Province"
+    },
+    {
+      "code": "CD-NK",
+      "name": "Nord-Kivu",
+      "type": "Province"
+    },
+    {
+      "code": "CD-NU",
+      "name": "Nord-Ubangi",
+      "type": "Province"
+    },
+    {
+      "code": "CD-SA",
+      "name": "Sankuru",
+      "type": "Province"
+    },
+    {
+      "code": "CD-SK",
+      "name": "Sud-Kivu",
+      "type": "Province"
+    },
+    {
+      "code": "CD-SU",
+      "name": "Sud-Ubangi",
+      "type": "Province"
+    },
+    {
+      "code": "CD-TA",
+      "name": "Tanganyika",
+      "type": "Province"
+    },
+    {
+      "code": "CD-TO",
+      "name": "Tshopo",
+      "type": "Province"
+    },
+    {
+      "code": "CD-TU",
+      "name": "Tshuapa",
+      "type": "Province"
+    },
+    {
+      "code": "CF-AC",
+      "name": "Ouham",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CF-BB",
+      "name": "Bamingui-Bangoran",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CF-BGF",
+      "name": "Bangui",
+      "type": "Commune"
+    },
+    {
+      "code": "CF-BK",
+      "name": "Basse-Kotto",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CF-HK",
+      "name": "Haute-Kotto",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CF-HM",
+      "name": "Haut-Mbomou",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CF-HS",
+      "name": "Haute-Sangha / Mambéré-Kadéï",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CF-KB",
+      "name": "Gribingui",
+      "type": "Economic prefecture"
+    },
+    {
+      "code": "CF-KG",
+      "name": "Kémo-Gribingui",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CF-LB",
+      "name": "Lobaye",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CF-MB",
+      "name": "Mbomou",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CF-MP",
+      "name": "Ombella-Mpoko",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CF-NM",
+      "name": "Nana-Mambéré",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CF-OP",
+      "name": "Ouham-Pendé",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CF-SE",
+      "name": "Sangha",
+      "type": "Economic prefecture"
+    },
+    {
+      "code": "CF-UK",
+      "name": "Ouaka",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CF-VK",
+      "name": "Vakaga",
+      "type": "Prefecture"
+    },
+    {
+      "code": "CG-11",
+      "name": "Bouenza",
+      "type": "Department"
+    },
+    {
+      "code": "CG-12",
+      "name": "Pool",
+      "type": "Department"
+    },
+    {
+      "code": "CG-13",
+      "name": "Sangha",
+      "type": "Department"
+    },
+    {
+      "code": "CG-14",
+      "name": "Plateaux",
+      "type": "Department"
+    },
+    {
+      "code": "CG-15",
+      "name": "Cuvette-Ouest",
+      "type": "Department"
+    },
+    {
+      "code": "CG-16",
+      "name": "Pointe-Noire",
+      "type": "Department"
+    },
+    {
+      "code": "CG-2",
+      "name": "Lékoumou",
+      "type": "Department"
+    },
+    {
+      "code": "CG-5",
+      "name": "Kouilou",
+      "type": "Department"
+    },
+    {
+      "code": "CG-7",
+      "name": "Likouala",
+      "type": "Department"
+    },
+    {
+      "code": "CG-8",
+      "name": "Cuvette",
+      "type": "Department"
+    },
+    {
+      "code": "CG-9",
+      "name": "Niari",
+      "type": "Department"
+    },
+    {
+      "code": "CG-BZV",
+      "name": "Brazzaville",
+      "type": "Department"
+    },
+    {
+      "code": "CH-AG",
+      "name": "Aargau",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-AI",
+      "name": "Appenzell Innerrhoden",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-AR",
+      "name": "Appenzell Ausserrhoden",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-BE",
+      "name": "Berne",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-BL",
+      "name": "Basel-Landschaft",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-BS",
+      "name": "Basel-Stadt",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-FR",
+      "name": "Fribourg",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-GE",
+      "name": "Genève",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-GL",
+      "name": "Glarus",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-GR",
+      "name": "Graubünden",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-JU",
+      "name": "Jura",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-LU",
+      "name": "Luzern",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-NE",
+      "name": "Neuchâtel",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-NW",
+      "name": "Nidwalden",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-OW",
+      "name": "Obwalden",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-SG",
+      "name": "Sankt Gallen",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-SH",
+      "name": "Schaffhausen",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-SO",
+      "name": "Solothurn",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-SZ",
+      "name": "Schwyz",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-TG",
+      "name": "Thurgau",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-TI",
+      "name": "Ticino",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-UR",
+      "name": "Uri",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-VD",
+      "name": "Vaud",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-VS",
+      "name": "Valais",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-ZG",
+      "name": "Zug",
+      "type": "Canton"
+    },
+    {
+      "code": "CH-ZH",
+      "name": "Zürich",
+      "type": "Canton"
+    },
+    {
+      "code": "CI-AB",
+      "name": "Abidjan",
+      "type": "Autonomous district"
+    },
+    {
+      "code": "CI-BS",
+      "name": "Bas-Sassandra",
+      "type": "District"
+    },
+    {
+      "code": "CI-CM",
+      "name": "Comoé",
+      "type": "District"
+    },
+    {
+      "code": "CI-DN",
+      "name": "Denguélé",
+      "type": "District"
+    },
+    {
+      "code": "CI-GD",
+      "name": "Gôh-Djiboua",
+      "type": "District"
+    },
+    {
+      "code": "CI-LC",
+      "name": "Lacs",
+      "type": "District"
+    },
+    {
+      "code": "CI-LG",
+      "name": "Lagunes",
+      "type": "District"
+    },
+    {
+      "code": "CI-MG",
+      "name": "Montagnes",
+      "type": "District"
+    },
+    {
+      "code": "CI-SM",
+      "name": "Sassandra-Marahoué",
+      "type": "District"
+    },
+    {
+      "code": "CI-SV",
+      "name": "Savanes",
+      "type": "District"
+    },
+    {
+      "code": "CI-VB",
+      "name": "Vallée du Bandama",
+      "type": "District"
+    },
+    {
+      "code": "CI-WR",
+      "name": "Woroba",
+      "type": "District"
+    },
+    {
+      "code": "CI-YM",
+      "name": "Yamoussoukro",
+      "type": "Autonomous district"
+    },
+    {
+      "code": "CI-ZZ",
+      "name": "Zanzan",
+      "type": "District"
+    },
+    {
+      "code": "CL-AI",
+      "name": "Aisén del General Carlos Ibañez del Campo",
+      "type": "Region"
+    },
+    {
+      "code": "CL-AN",
+      "name": "Antofagasta",
+      "type": "Region"
+    },
+    {
+      "code": "CL-AP",
+      "name": "Arica y Parinacota",
+      "type": "Region"
+    },
+    {
+      "code": "CL-AR",
+      "name": "La Araucanía",
+      "type": "Region"
+    },
+    {
+      "code": "CL-AT",
+      "name": "Atacama",
+      "type": "Region"
+    },
+    {
+      "code": "CL-BI",
+      "name": "Biobío",
+      "type": "Region"
+    },
+    {
+      "code": "CL-CO",
+      "name": "Coquimbo",
+      "type": "Region"
+    },
+    {
+      "code": "CL-LI",
+      "name": "Libertador General Bernardo O'Higgins",
+      "type": "Region"
+    },
+    {
+      "code": "CL-LL",
+      "name": "Los Lagos",
+      "type": "Region"
+    },
+    {
+      "code": "CL-LR",
+      "name": "Los Ríos",
+      "type": "Region"
+    },
+    {
+      "code": "CL-MA",
+      "name": "Magallanes",
+      "type": "Region"
+    },
+    {
+      "code": "CL-ML",
+      "name": "Maule",
+      "type": "Region"
+    },
+    {
+      "code": "CL-NB",
+      "name": "Ñuble",
+      "type": "Region"
+    },
+    {
+      "code": "CL-RM",
+      "name": "Región Metropolitana de Santiago",
+      "type": "Region"
+    },
+    {
+      "code": "CL-TA",
+      "name": "Tarapacá",
+      "type": "Region"
+    },
+    {
+      "code": "CL-VS",
+      "name": "Valparaíso",
+      "type": "Region"
+    },
+    {
+      "code": "CM-AD",
+      "name": "Adamaoua",
+      "type": "Region"
+    },
+    {
+      "code": "CM-CE",
+      "name": "Centre",
+      "type": "Region"
+    },
+    {
+      "code": "CM-EN",
+      "name": "Far North",
+      "type": "Region"
+    },
+    {
+      "code": "CM-ES",
+      "name": "East",
+      "type": "Region"
+    },
+    {
+      "code": "CM-LT",
+      "name": "Littoral",
+      "type": "Region"
+    },
+    {
+      "code": "CM-NO",
+      "name": "North",
+      "type": "Region"
+    },
+    {
+      "code": "CM-NW",
+      "name": "North-West",
+      "type": "Region"
+    },
+    {
+      "code": "CM-OU",
+      "name": "West",
+      "type": "Region"
+    },
+    {
+      "code": "CM-SU",
+      "name": "South",
+      "type": "Region"
+    },
+    {
+      "code": "CM-SW",
+      "name": "South-West",
+      "type": "Region"
+    },
+    {
+      "code": "CN-AH",
+      "name": "Anhui Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-BJ",
+      "name": "Beijing Shi",
+      "type": "Municipality"
+    },
+    {
+      "code": "CN-CQ",
+      "name": "Chongqing Shi",
+      "type": "Municipality"
+    },
+    {
+      "code": "CN-FJ",
+      "name": "Fujian Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-GD",
+      "name": "Guangdong Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-GS",
+      "name": "Gansu Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-GX",
+      "name": "Guangxi Zhuangzu Zizhiqu",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "CN-GZ",
+      "name": "Guizhou Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-HA",
+      "name": "Henan Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-HB",
+      "name": "Hubei Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-HE",
+      "name": "Hebei Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-HI",
+      "name": "Hainan Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-HK",
+      "name": "Hong Kong SAR",
+      "type": "Special administrative region"
+    },
+    {
+      "code": "CN-HL",
+      "name": "Heilongjiang Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-HN",
+      "name": "Hunan Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-JL",
+      "name": "Jilin Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-JS",
+      "name": "Jiangsu Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-JX",
+      "name": "Jiangxi Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-LN",
+      "name": "Liaoning Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-MO",
+      "name": "Macao SAR",
+      "type": "Special administrative region"
+    },
+    {
+      "code": "CN-NM",
+      "name": "Nei Mongol Zizhiqu",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "CN-NX",
+      "name": "Ningxia Huizu Zizhiqu",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "CN-QH",
+      "name": "Qinghai Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-SC",
+      "name": "Sichuan Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-SD",
+      "name": "Shandong Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-SH",
+      "name": "Shanghai Shi",
+      "type": "Municipality"
+    },
+    {
+      "code": "CN-SN",
+      "name": "Shaanxi Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-SX",
+      "name": "Shanxi Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-TJ",
+      "name": "Tianjin Shi",
+      "type": "Municipality"
+    },
+    {
+      "code": "CN-TW",
+      "name": "Taiwan Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-XJ",
+      "name": "Xinjiang Uygur Zizhiqu",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "CN-XZ",
+      "name": "Xizang Zizhiqu",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "CN-YN",
+      "name": "Yunnan Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CN-ZJ",
+      "name": "Zhejiang Sheng",
+      "type": "Province"
+    },
+    {
+      "code": "CO-AMA",
+      "name": "Amazonas",
+      "type": "Department"
+    },
+    {
+      "code": "CO-ANT",
+      "name": "Antioquia",
+      "type": "Department"
+    },
+    {
+      "code": "CO-ARA",
+      "name": "Arauca",
+      "type": "Department"
+    },
+    {
+      "code": "CO-ATL",
+      "name": "Atlántico",
+      "type": "Department"
+    },
+    {
+      "code": "CO-BOL",
+      "name": "Bolívar",
+      "type": "Department"
+    },
+    {
+      "code": "CO-BOY",
+      "name": "Boyacá",
+      "type": "Department"
+    },
+    {
+      "code": "CO-CAL",
+      "name": "Caldas",
+      "type": "Department"
+    },
+    {
+      "code": "CO-CAQ",
+      "name": "Caquetá",
+      "type": "Department"
+    },
+    {
+      "code": "CO-CAS",
+      "name": "Casanare",
+      "type": "Department"
+    },
+    {
+      "code": "CO-CAU",
+      "name": "Cauca",
+      "type": "Department"
+    },
+    {
+      "code": "CO-CES",
+      "name": "Cesar",
+      "type": "Department"
+    },
+    {
+      "code": "CO-CHO",
+      "name": "Chocó",
+      "type": "Department"
+    },
+    {
+      "code": "CO-COR",
+      "name": "Córdoba",
+      "type": "Department"
+    },
+    {
+      "code": "CO-CUN",
+      "name": "Cundinamarca",
+      "type": "Department"
+    },
+    {
+      "code": "CO-DC",
+      "name": "Distrito Capital de Bogotá",
+      "type": "Capital district"
+    },
+    {
+      "code": "CO-GUA",
+      "name": "Guainía",
+      "type": "Department"
+    },
+    {
+      "code": "CO-GUV",
+      "name": "Guaviare",
+      "type": "Department"
+    },
+    {
+      "code": "CO-HUI",
+      "name": "Huila",
+      "type": "Department"
+    },
+    {
+      "code": "CO-LAG",
+      "name": "La Guajira",
+      "type": "Department"
+    },
+    {
+      "code": "CO-MAG",
+      "name": "Magdalena",
+      "type": "Department"
+    },
+    {
+      "code": "CO-MET",
+      "name": "Meta",
+      "type": "Department"
+    },
+    {
+      "code": "CO-NAR",
+      "name": "Nariño",
+      "type": "Department"
+    },
+    {
+      "code": "CO-NSA",
+      "name": "Norte de Santander",
+      "type": "Department"
+    },
+    {
+      "code": "CO-PUT",
+      "name": "Putumayo",
+      "type": "Department"
+    },
+    {
+      "code": "CO-QUI",
+      "name": "Quindío",
+      "type": "Department"
+    },
+    {
+      "code": "CO-RIS",
+      "name": "Risaralda",
+      "type": "Department"
+    },
+    {
+      "code": "CO-SAN",
+      "name": "Santander",
+      "type": "Department"
+    },
+    {
+      "code": "CO-SAP",
+      "name": "San Andrés, Providencia y Santa Catalina",
+      "type": "Department"
+    },
+    {
+      "code": "CO-SUC",
+      "name": "Sucre",
+      "type": "Department"
+    },
+    {
+      "code": "CO-TOL",
+      "name": "Tolima",
+      "type": "Department"
+    },
+    {
+      "code": "CO-VAC",
+      "name": "Valle del Cauca",
+      "type": "Department"
+    },
+    {
+      "code": "CO-VAU",
+      "name": "Vaupés",
+      "type": "Department"
+    },
+    {
+      "code": "CO-VID",
+      "name": "Vichada",
+      "type": "Department"
+    },
+    {
+      "code": "CR-A",
+      "name": "Alajuela",
+      "type": "Province"
+    },
+    {
+      "code": "CR-C",
+      "name": "Cartago",
+      "type": "Province"
+    },
+    {
+      "code": "CR-G",
+      "name": "Guanacaste",
+      "type": "Province"
+    },
+    {
+      "code": "CR-H",
+      "name": "Heredia",
+      "type": "Province"
+    },
+    {
+      "code": "CR-L",
+      "name": "Limón",
+      "type": "Province"
+    },
+    {
+      "code": "CR-P",
+      "name": "Puntarenas",
+      "type": "Province"
+    },
+    {
+      "code": "CR-SJ",
+      "name": "San José",
+      "type": "Province"
+    },
+    {
+      "code": "CU-01",
+      "name": "Pinar del Río",
+      "type": "Province"
+    },
+    {
+      "code": "CU-03",
+      "name": "La Habana",
+      "type": "Province"
+    },
+    {
+      "code": "CU-04",
+      "name": "Matanzas",
+      "type": "Province"
+    },
+    {
+      "code": "CU-05",
+      "name": "Villa Clara",
+      "type": "Province"
+    },
+    {
+      "code": "CU-06",
+      "name": "Cienfuegos",
+      "type": "Province"
+    },
+    {
+      "code": "CU-07",
+      "name": "Sancti Spíritus",
+      "type": "Province"
+    },
+    {
+      "code": "CU-08",
+      "name": "Ciego de Ávila",
+      "type": "Province"
+    },
+    {
+      "code": "CU-09",
+      "name": "Camagüey",
+      "type": "Province"
+    },
+    {
+      "code": "CU-10",
+      "name": "Las Tunas",
+      "type": "Province"
+    },
+    {
+      "code": "CU-11",
+      "name": "Holguín",
+      "type": "Province"
+    },
+    {
+      "code": "CU-12",
+      "name": "Granma",
+      "type": "Province"
+    },
+    {
+      "code": "CU-13",
+      "name": "Santiago de Cuba",
+      "type": "Province"
+    },
+    {
+      "code": "CU-14",
+      "name": "Guantánamo",
+      "type": "Province"
+    },
+    {
+      "code": "CU-15",
+      "name": "Artemisa",
+      "type": "Province"
+    },
+    {
+      "code": "CU-16",
+      "name": "Mayabeque",
+      "type": "Province"
+    },
+    {
+      "code": "CU-99",
+      "name": "Isla de la Juventud",
+      "type": "Special municipality"
+    },
+    {
+      "code": "CV-B",
+      "name": "Ilhas de Barlavento",
+      "type": "Geographical region"
+    },
+    {
+      "code": "CV-BR",
+      "name": "Brava",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-BV",
+      "name": "Boa Vista",
+      "parent": "CV-B",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-CA",
+      "name": "Santa Catarina",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-CF",
+      "name": "Santa Catarina do Fogo",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-CR",
+      "name": "Santa Cruz",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-MA",
+      "name": "Maio",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-MO",
+      "name": "Mosteiros",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-PA",
+      "name": "Paul",
+      "parent": "CV-B",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-PN",
+      "name": "Porto Novo",
+      "parent": "CV-B",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-PR",
+      "name": "Praia",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-RB",
+      "name": "Ribeira Brava",
+      "parent": "CV-B",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-RG",
+      "name": "Ribeira Grande",
+      "parent": "CV-B",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-RS",
+      "name": "Ribeira Grande de Santiago",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-S",
+      "name": "Ilhas de Sotavento",
+      "type": "Geographical region"
+    },
+    {
+      "code": "CV-SD",
+      "name": "São Domingos",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-SF",
+      "name": "São Filipe",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-SL",
+      "name": "Sal",
+      "parent": "CV-B",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-SM",
+      "name": "São Miguel",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-SO",
+      "name": "São Lourenço dos Órgãos",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-SS",
+      "name": "São Salvador do Mundo",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-SV",
+      "name": "São Vicente",
+      "parent": "CV-B",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-TA",
+      "name": "Tarrafal",
+      "parent": "CV-S",
+      "type": "Municipality"
+    },
+    {
+      "code": "CV-TS",
+      "name": "Tarrafal de São Nicolau",
+      "parent": "CV-B",
+      "type": "Municipality"
+    },
+    {
+      "code": "CY-01",
+      "name": "Lefkosia",
+      "type": "District"
+    },
+    {
+      "code": "CY-02",
+      "name": "Lemesos",
+      "type": "District"
+    },
+    {
+      "code": "CY-03",
+      "name": "Larnaka",
+      "type": "District"
+    },
+    {
+      "code": "CY-04",
+      "name": "Ammochostos",
+      "type": "District"
+    },
+    {
+      "code": "CY-05",
+      "name": "Baf",
+      "type": "District"
+    },
+    {
+      "code": "CY-06",
+      "name": "Girne",
+      "type": "District"
+    },
+    {
+      "code": "CZ-10",
+      "name": "Praha, Hlavní město",
+      "type": "Capital city"
+    },
+    {
+      "code": "CZ-20",
+      "name": "Středočeský kraj",
+      "type": "Region"
+    },
+    {
+      "code": "CZ-201",
+      "name": "Benešov",
+      "parent": "CZ-20",
+      "type": "District"
+    },
+    {
+      "code": "CZ-202",
+      "name": "Beroun",
+      "parent": "CZ-20",
+      "type": "District"
+    },
+    {
+      "code": "CZ-203",
+      "name": "Kladno",
+      "parent": "CZ-20",
+      "type": "District"
+    },
+    {
+      "code": "CZ-204",
+      "name": "Kolín",
+      "parent": "CZ-20",
+      "type": "District"
+    },
+    {
+      "code": "CZ-205",
+      "name": "Kutná Hora",
+      "parent": "CZ-20",
+      "type": "District"
+    },
+    {
+      "code": "CZ-206",
+      "name": "Mělník",
+      "parent": "CZ-20",
+      "type": "District"
+    },
+    {
+      "code": "CZ-207",
+      "name": "Mladá Boleslav",
+      "parent": "CZ-20",
+      "type": "District"
+    },
+    {
+      "code": "CZ-208",
+      "name": "Nymburk",
+      "parent": "CZ-20",
+      "type": "District"
+    },
+    {
+      "code": "CZ-209",
+      "name": "Praha-východ",
+      "parent": "CZ-20",
+      "type": "District"
+    },
+    {
+      "code": "CZ-20A",
+      "name": "Praha-západ",
+      "parent": "CZ-20",
+      "type": "District"
+    },
+    {
+      "code": "CZ-20B",
+      "name": "Příbram",
+      "parent": "CZ-20",
+      "type": "District"
+    },
+    {
+      "code": "CZ-20C",
+      "name": "Rakovník",
+      "parent": "CZ-20",
+      "type": "District"
+    },
+    {
+      "code": "CZ-31",
+      "name": "Jihočeský kraj",
+      "type": "Region"
+    },
+    {
+      "code": "CZ-311",
+      "name": "České Budějovice",
+      "parent": "CZ-31",
+      "type": "District"
+    },
+    {
+      "code": "CZ-312",
+      "name": "Český Krumlov",
+      "parent": "CZ-31",
+      "type": "District"
+    },
+    {
+      "code": "CZ-313",
+      "name": "Jindřichův Hradec",
+      "parent": "CZ-31",
+      "type": "District"
+    },
+    {
+      "code": "CZ-314",
+      "name": "Písek",
+      "parent": "CZ-31",
+      "type": "District"
+    },
+    {
+      "code": "CZ-315",
+      "name": "Prachatice",
+      "parent": "CZ-31",
+      "type": "District"
+    },
+    {
+      "code": "CZ-316",
+      "name": "Strakonice",
+      "parent": "CZ-31",
+      "type": "District"
+    },
+    {
+      "code": "CZ-317",
+      "name": "Tábor",
+      "parent": "CZ-31",
+      "type": "District"
+    },
+    {
+      "code": "CZ-32",
+      "name": "Plzeňský kraj",
+      "type": "Region"
+    },
+    {
+      "code": "CZ-321",
+      "name": "Domažlice",
+      "parent": "CZ-32",
+      "type": "District"
+    },
+    {
+      "code": "CZ-322",
+      "name": "Klatovy",
+      "parent": "CZ-32",
+      "type": "District"
+    },
+    {
+      "code": "CZ-323",
+      "name": "Plzeň-město",
+      "parent": "CZ-32",
+      "type": "District"
+    },
+    {
+      "code": "CZ-324",
+      "name": "Plzeň-jih",
+      "parent": "CZ-32",
+      "type": "District"
+    },
+    {
+      "code": "CZ-325",
+      "name": "Plzeň-sever",
+      "parent": "CZ-32",
+      "type": "District"
+    },
+    {
+      "code": "CZ-326",
+      "name": "Rokycany",
+      "parent": "CZ-32",
+      "type": "District"
+    },
+    {
+      "code": "CZ-327",
+      "name": "Tachov",
+      "parent": "CZ-32",
+      "type": "District"
+    },
+    {
+      "code": "CZ-41",
+      "name": "Karlovarský kraj",
+      "type": "Region"
+    },
+    {
+      "code": "CZ-411",
+      "name": "Cheb",
+      "parent": "CZ-41",
+      "type": "District"
+    },
+    {
+      "code": "CZ-412",
+      "name": "Karlovy Vary",
+      "parent": "CZ-41",
+      "type": "District"
+    },
+    {
+      "code": "CZ-413",
+      "name": "Sokolov",
+      "parent": "CZ-41",
+      "type": "District"
+    },
+    {
+      "code": "CZ-42",
+      "name": "Ústecký kraj",
+      "type": "Region"
+    },
+    {
+      "code": "CZ-421",
+      "name": "Děčín",
+      "parent": "CZ-42",
+      "type": "District"
+    },
+    {
+      "code": "CZ-422",
+      "name": "Chomutov",
+      "parent": "CZ-42",
+      "type": "District"
+    },
+    {
+      "code": "CZ-423",
+      "name": "Litoměřice",
+      "parent": "CZ-42",
+      "type": "District"
+    },
+    {
+      "code": "CZ-424",
+      "name": "Louny",
+      "parent": "CZ-42",
+      "type": "District"
+    },
+    {
+      "code": "CZ-425",
+      "name": "Most",
+      "parent": "CZ-42",
+      "type": "District"
+    },
+    {
+      "code": "CZ-426",
+      "name": "Teplice",
+      "parent": "CZ-42",
+      "type": "District"
+    },
+    {
+      "code": "CZ-427",
+      "name": "Ústí nad Labem",
+      "parent": "CZ-42",
+      "type": "District"
+    },
+    {
+      "code": "CZ-51",
+      "name": "Liberecký kraj",
+      "type": "Region"
+    },
+    {
+      "code": "CZ-511",
+      "name": "Česká Lípa",
+      "parent": "CZ-51",
+      "type": "District"
+    },
+    {
+      "code": "CZ-512",
+      "name": "Jablonec nad Nisou",
+      "parent": "CZ-51",
+      "type": "District"
+    },
+    {
+      "code": "CZ-513",
+      "name": "Liberec",
+      "parent": "CZ-51",
+      "type": "District"
+    },
+    {
+      "code": "CZ-514",
+      "name": "Semily",
+      "parent": "CZ-51",
+      "type": "District"
+    },
+    {
+      "code": "CZ-52",
+      "name": "Královéhradecký kraj",
+      "type": "Region"
+    },
+    {
+      "code": "CZ-521",
+      "name": "Hradec Králové",
+      "parent": "CZ-52",
+      "type": "District"
+    },
+    {
+      "code": "CZ-522",
+      "name": "Jičín",
+      "parent": "CZ-52",
+      "type": "District"
+    },
+    {
+      "code": "CZ-523",
+      "name": "Náchod",
+      "parent": "CZ-52",
+      "type": "District"
+    },
+    {
+      "code": "CZ-524",
+      "name": "Rychnov nad Kněžnou",
+      "parent": "CZ-52",
+      "type": "District"
+    },
+    {
+      "code": "CZ-525",
+      "name": "Trutnov",
+      "parent": "CZ-52",
+      "type": "District"
+    },
+    {
+      "code": "CZ-53",
+      "name": "Pardubický kraj",
+      "type": "Region"
+    },
+    {
+      "code": "CZ-531",
+      "name": "Chrudim",
+      "parent": "CZ-53",
+      "type": "District"
+    },
+    {
+      "code": "CZ-532",
+      "name": "Pardubice",
+      "parent": "CZ-53",
+      "type": "District"
+    },
+    {
+      "code": "CZ-533",
+      "name": "Svitavy",
+      "parent": "CZ-53",
+      "type": "District"
+    },
+    {
+      "code": "CZ-534",
+      "name": "Ústí nad Orlicí",
+      "parent": "CZ-53",
+      "type": "District"
+    },
+    {
+      "code": "CZ-63",
+      "name": "Kraj Vysočina",
+      "type": "Region"
+    },
+    {
+      "code": "CZ-631",
+      "name": "Havlíčkův Brod",
+      "parent": "CZ-63",
+      "type": "District"
+    },
+    {
+      "code": "CZ-632",
+      "name": "Jihlava",
+      "parent": "CZ-63",
+      "type": "District"
+    },
+    {
+      "code": "CZ-633",
+      "name": "Pelhřimov",
+      "parent": "CZ-63",
+      "type": "District"
+    },
+    {
+      "code": "CZ-634",
+      "name": "Třebíč",
+      "parent": "CZ-63",
+      "type": "District"
+    },
+    {
+      "code": "CZ-635",
+      "name": "Žďár nad Sázavou",
+      "parent": "CZ-63",
+      "type": "District"
+    },
+    {
+      "code": "CZ-64",
+      "name": "Jihomoravský kraj",
+      "type": "Region"
+    },
+    {
+      "code": "CZ-641",
+      "name": "Blansko",
+      "parent": "CZ-64",
+      "type": "District"
+    },
+    {
+      "code": "CZ-642",
+      "name": "Brno-město",
+      "parent": "CZ-64",
+      "type": "District"
+    },
+    {
+      "code": "CZ-643",
+      "name": "Brno-venkov",
+      "parent": "CZ-64",
+      "type": "District"
+    },
+    {
+      "code": "CZ-644",
+      "name": "Břeclav",
+      "parent": "CZ-64",
+      "type": "District"
+    },
+    {
+      "code": "CZ-645",
+      "name": "Hodonín",
+      "parent": "CZ-64",
+      "type": "District"
+    },
+    {
+      "code": "CZ-646",
+      "name": "Vyškov",
+      "parent": "CZ-64",
+      "type": "District"
+    },
+    {
+      "code": "CZ-647",
+      "name": "Znojmo",
+      "parent": "CZ-64",
+      "type": "District"
+    },
+    {
+      "code": "CZ-71",
+      "name": "Olomoucký kraj",
+      "type": "Region"
+    },
+    {
+      "code": "CZ-711",
+      "name": "Jeseník",
+      "parent": "CZ-71",
+      "type": "District"
+    },
+    {
+      "code": "CZ-712",
+      "name": "Olomouc",
+      "parent": "CZ-71",
+      "type": "District"
+    },
+    {
+      "code": "CZ-713",
+      "name": "Prostějov",
+      "parent": "CZ-71",
+      "type": "District"
+    },
+    {
+      "code": "CZ-714",
+      "name": "Přerov",
+      "parent": "CZ-71",
+      "type": "District"
+    },
+    {
+      "code": "CZ-715",
+      "name": "Šumperk",
+      "parent": "CZ-71",
+      "type": "District"
+    },
+    {
+      "code": "CZ-72",
+      "name": "Zlínský kraj",
+      "type": "Region"
+    },
+    {
+      "code": "CZ-721",
+      "name": "Kroměříž",
+      "parent": "CZ-72",
+      "type": "District"
+    },
+    {
+      "code": "CZ-722",
+      "name": "Uherské Hradiště",
+      "parent": "CZ-72",
+      "type": "District"
+    },
+    {
+      "code": "CZ-723",
+      "name": "Vsetín",
+      "parent": "CZ-72",
+      "type": "District"
+    },
+    {
+      "code": "CZ-724",
+      "name": "Zlín",
+      "parent": "CZ-72",
+      "type": "District"
+    },
+    {
+      "code": "CZ-80",
+      "name": "Moravskoslezský kraj",
+      "type": "Region"
+    },
+    {
+      "code": "CZ-801",
+      "name": "Bruntál",
+      "parent": "CZ-80",
+      "type": "District"
+    },
+    {
+      "code": "CZ-802",
+      "name": "Frýdek-Místek",
+      "parent": "CZ-80",
+      "type": "District"
+    },
+    {
+      "code": "CZ-803",
+      "name": "Karviná",
+      "parent": "CZ-80",
+      "type": "District"
+    },
+    {
+      "code": "CZ-804",
+      "name": "Nový Jičín",
+      "parent": "CZ-80",
+      "type": "District"
+    },
+    {
+      "code": "CZ-805",
+      "name": "Opava",
+      "parent": "CZ-80",
+      "type": "District"
+    },
+    {
+      "code": "CZ-806",
+      "name": "Ostrava-město",
+      "parent": "CZ-80",
+      "type": "District"
+    },
+    {
+      "code": "DE-BB",
+      "name": "Brandenburg",
+      "type": "Land"
+    },
+    {
+      "code": "DE-BE",
+      "name": "Berlin",
+      "type": "Land"
+    },
+    {
+      "code": "DE-BW",
+      "name": "Baden-Württemberg",
+      "type": "Land"
+    },
+    {
+      "code": "DE-BY",
+      "name": "Bayern",
+      "type": "Land"
+    },
+    {
+      "code": "DE-HB",
+      "name": "Bremen",
+      "type": "Land"
+    },
+    {
+      "code": "DE-HE",
+      "name": "Hessen",
+      "type": "Land"
+    },
+    {
+      "code": "DE-HH",
+      "name": "Hamburg",
+      "type": "Land"
+    },
+    {
+      "code": "DE-MV",
+      "name": "Mecklenburg-Vorpommern",
+      "type": "Land"
+    },
+    {
+      "code": "DE-NI",
+      "name": "Niedersachsen",
+      "type": "Land"
+    },
+    {
+      "code": "DE-NW",
+      "name": "Nordrhein-Westfalen",
+      "type": "Land"
+    },
+    {
+      "code": "DE-RP",
+      "name": "Rheinland-Pfalz",
+      "type": "Land"
+    },
+    {
+      "code": "DE-SH",
+      "name": "Schleswig-Holstein",
+      "type": "Land"
+    },
+    {
+      "code": "DE-SL",
+      "name": "Saarland",
+      "type": "Land"
+    },
+    {
+      "code": "DE-SN",
+      "name": "Sachsen",
+      "type": "Land"
+    },
+    {
+      "code": "DE-ST",
+      "name": "Sachsen-Anhalt",
+      "type": "Land"
+    },
+    {
+      "code": "DE-TH",
+      "name": "Thüringen",
+      "type": "Land"
+    },
+    {
+      "code": "DJ-AR",
+      "name": "Arta",
+      "type": "Region"
+    },
+    {
+      "code": "DJ-AS",
+      "name": "Ali Sabieh",
+      "type": "Region"
+    },
+    {
+      "code": "DJ-DI",
+      "name": "Dikhil",
+      "type": "Region"
+    },
+    {
+      "code": "DJ-DJ",
+      "name": "Djibouti",
+      "type": "City"
+    },
+    {
+      "code": "DJ-OB",
+      "name": "Obock",
+      "type": "Region"
+    },
+    {
+      "code": "DJ-TA",
+      "name": "Tadjourah",
+      "type": "Region"
+    },
+    {
+      "code": "DK-81",
+      "name": "Nordjylland",
+      "type": "Region"
+    },
+    {
+      "code": "DK-82",
+      "name": "Midtjylland",
+      "type": "Region"
+    },
+    {
+      "code": "DK-83",
+      "name": "Syddanmark",
+      "type": "Region"
+    },
+    {
+      "code": "DK-84",
+      "name": "Hovedstaden",
+      "type": "Region"
+    },
+    {
+      "code": "DK-85",
+      "name": "Sjælland",
+      "type": "Region"
+    },
+    {
+      "code": "DM-02",
+      "name": "Saint Andrew",
+      "type": "Parish"
+    },
+    {
+      "code": "DM-03",
+      "name": "Saint David",
+      "type": "Parish"
+    },
+    {
+      "code": "DM-04",
+      "name": "Saint George",
+      "type": "Parish"
+    },
+    {
+      "code": "DM-05",
+      "name": "Saint John",
+      "type": "Parish"
+    },
+    {
+      "code": "DM-06",
+      "name": "Saint Joseph",
+      "type": "Parish"
+    },
+    {
+      "code": "DM-07",
+      "name": "Saint Luke",
+      "type": "Parish"
+    },
+    {
+      "code": "DM-08",
+      "name": "Saint Mark",
+      "type": "Parish"
+    },
+    {
+      "code": "DM-09",
+      "name": "Saint Patrick",
+      "type": "Parish"
+    },
+    {
+      "code": "DM-10",
+      "name": "Saint Paul",
+      "type": "Parish"
+    },
+    {
+      "code": "DM-11",
+      "name": "Saint Peter",
+      "type": "Parish"
+    },
+    {
+      "code": "DO-01",
+      "name": "Distrito Nacional (Santo Domingo)",
+      "parent": "DO-40",
+      "type": "District"
+    },
+    {
+      "code": "DO-02",
+      "name": "Azua",
+      "parent": "DO-41",
+      "type": "Province"
+    },
+    {
+      "code": "DO-03",
+      "name": "Baoruco",
+      "parent": "DO-38",
+      "type": "Province"
+    },
+    {
+      "code": "DO-04",
+      "name": "Barahona",
+      "parent": "DO-38",
+      "type": "Province"
+    },
+    {
+      "code": "DO-05",
+      "name": "Dajabón",
+      "parent": "DO-34",
+      "type": "Province"
+    },
+    {
+      "code": "DO-06",
+      "name": "Duarte",
+      "parent": "DO-33",
+      "type": "Province"
+    },
+    {
+      "code": "DO-07",
+      "name": "Elías Piña",
+      "parent": "DO-37",
+      "type": "Province"
+    },
+    {
+      "code": "DO-08",
+      "name": "El Seibo",
+      "parent": "DO-42",
+      "type": "Province"
+    },
+    {
+      "code": "DO-09",
+      "name": "Espaillat",
+      "parent": "DO-35",
+      "type": "Province"
+    },
+    {
+      "code": "DO-10",
+      "name": "Independencia",
+      "parent": "DO-38",
+      "type": "Province"
+    },
+    {
+      "code": "DO-11",
+      "name": "La Altagracia",
+      "parent": "DO-42",
+      "type": "Province"
+    },
+    {
+      "code": "DO-12",
+      "name": "La Romana",
+      "parent": "DO-42",
+      "type": "Province"
+    },
+    {
+      "code": "DO-13",
+      "name": "La Vega",
+      "parent": "DO-36",
+      "type": "Province"
+    },
+    {
+      "code": "DO-14",
+      "name": "María Trinidad Sánchez",
+      "parent": "DO-33",
+      "type": "Province"
+    },
+    {
+      "code": "DO-15",
+      "name": "Monte Cristi",
+      "parent": "DO-34",
+      "type": "Province"
+    },
+    {
+      "code": "DO-16",
+      "name": "Pedernales",
+      "parent": "DO-38",
+      "type": "Province"
+    },
+    {
+      "code": "DO-17",
+      "name": "Peravia",
+      "parent": "DO-41",
+      "type": "Province"
+    },
+    {
+      "code": "DO-18",
+      "name": "Puerto Plata",
+      "parent": "DO-35",
+      "type": "Province"
+    },
+    {
+      "code": "DO-19",
+      "name": "Hermanas Mirabal",
+      "parent": "DO-33",
+      "type": "Province"
+    },
+    {
+      "code": "DO-20",
+      "name": "Samaná",
+      "parent": "DO-33",
+      "type": "Province"
+    },
+    {
+      "code": "DO-21",
+      "name": "San Cristóbal",
+      "parent": "DO-41",
+      "type": "Province"
+    },
+    {
+      "code": "DO-22",
+      "name": "San Juan",
+      "parent": "DO-37",
+      "type": "Province"
+    },
+    {
+      "code": "DO-23",
+      "name": "San Pedro de Macorís",
+      "parent": "DO-39",
+      "type": "Province"
+    },
+    {
+      "code": "DO-24",
+      "name": "Sánchez Ramírez",
+      "parent": "DO-36",
+      "type": "Province"
+    },
+    {
+      "code": "DO-25",
+      "name": "Santiago",
+      "parent": "DO-35",
+      "type": "Province"
+    },
+    {
+      "code": "DO-26",
+      "name": "Santiago Rodríguez",
+      "parent": "DO-34",
+      "type": "Province"
+    },
+    {
+      "code": "DO-27",
+      "name": "Valverde",
+      "parent": "DO-34",
+      "type": "Province"
+    },
+    {
+      "code": "DO-28",
+      "name": "Monseñor Nouel",
+      "parent": "DO-36",
+      "type": "Province"
+    },
+    {
+      "code": "DO-29",
+      "name": "Monte Plata",
+      "parent": "DO-39",
+      "type": "Province"
+    },
+    {
+      "code": "DO-30",
+      "name": "Hato Mayor",
+      "parent": "DO-39",
+      "type": "Province"
+    },
+    {
+      "code": "DO-31",
+      "name": "San José de Ocoa",
+      "parent": "DO-41",
+      "type": "Province"
+    },
+    {
+      "code": "DO-32",
+      "name": "Santo Domingo",
+      "parent": "DO-40",
+      "type": "Province"
+    },
+    {
+      "code": "DO-33",
+      "name": "Cibao Nordeste",
+      "type": "Region"
+    },
+    {
+      "code": "DO-34",
+      "name": "Cibao Noroeste",
+      "type": "Region"
+    },
+    {
+      "code": "DO-35",
+      "name": "Cibao Norte",
+      "type": "Region"
+    },
+    {
+      "code": "DO-36",
+      "name": "Cibao Sur",
+      "type": "Region"
+    },
+    {
+      "code": "DO-37",
+      "name": "El Valle",
+      "type": "Region"
+    },
+    {
+      "code": "DO-38",
+      "name": "Enriquillo",
+      "type": "Region"
+    },
+    {
+      "code": "DO-39",
+      "name": "Higuamo",
+      "type": "Region"
+    },
+    {
+      "code": "DO-40",
+      "name": "Ozama",
+      "type": "Region"
+    },
+    {
+      "code": "DO-41",
+      "name": "Valdesia",
+      "type": "Region"
+    },
+    {
+      "code": "DO-42",
+      "name": "Yuma",
+      "type": "Region"
+    },
+    {
+      "code": "DZ-01",
+      "name": "Adrar",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-02",
+      "name": "Chlef",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-03",
+      "name": "Laghouat",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-04",
+      "name": "Oum el Bouaghi",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-05",
+      "name": "Batna",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-06",
+      "name": "Béjaïa",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-07",
+      "name": "Biskra",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-08",
+      "name": "Béchar",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-09",
+      "name": "Blida",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-10",
+      "name": "Bouira",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-11",
+      "name": "Tamanrasset",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-12",
+      "name": "Tébessa",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-13",
+      "name": "Tlemcen",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-14",
+      "name": "Tiaret",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-15",
+      "name": "Tizi Ouzou",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-16",
+      "name": "Alger",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-17",
+      "name": "Djelfa",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-18",
+      "name": "Jijel",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-19",
+      "name": "Sétif",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-20",
+      "name": "Saïda",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-21",
+      "name": "Skikda",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-22",
+      "name": "Sidi Bel Abbès",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-23",
+      "name": "Annaba",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-24",
+      "name": "Guelma",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-25",
+      "name": "Constantine",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-26",
+      "name": "Médéa",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-27",
+      "name": "Mostaganem",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-28",
+      "name": "M'sila",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-29",
+      "name": "Mascara",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-30",
+      "name": "Ouargla",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-31",
+      "name": "Oran",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-32",
+      "name": "El Bayadh",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-33",
+      "name": "Illizi",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-34",
+      "name": "Bordj Bou Arréridj",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-35",
+      "name": "Boumerdès",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-36",
+      "name": "El Tarf",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-37",
+      "name": "Tindouf",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-38",
+      "name": "Tissemsilt",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-39",
+      "name": "El Oued",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-40",
+      "name": "Khenchela",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-41",
+      "name": "Souk Ahras",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-42",
+      "name": "Tipaza",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-43",
+      "name": "Mila",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-44",
+      "name": "Aïn Defla",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-45",
+      "name": "Naama",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-46",
+      "name": "Aïn Témouchent",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-47",
+      "name": "Ghardaïa",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-48",
+      "name": "Relizane",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-49",
+      "name": "Timimoun",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-50",
+      "name": "Bordj Badji Mokhtar",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-51",
+      "name": "Ouled Djellal",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-52",
+      "name": "Béni Abbès",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-53",
+      "name": "In Salah",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-54",
+      "name": "In Guezzam",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-55",
+      "name": "Touggourt",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-56",
+      "name": "Djanet",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-57",
+      "name": "El Meghaier",
+      "type": "Province"
+    },
+    {
+      "code": "DZ-58",
+      "name": "El Meniaa",
+      "type": "Province"
+    },
+    {
+      "code": "EC-A",
+      "name": "Azuay",
+      "type": "Province"
+    },
+    {
+      "code": "EC-B",
+      "name": "Bolívar",
+      "type": "Province"
+    },
+    {
+      "code": "EC-C",
+      "name": "Carchi",
+      "type": "Province"
+    },
+    {
+      "code": "EC-D",
+      "name": "Orellana",
+      "type": "Province"
+    },
+    {
+      "code": "EC-E",
+      "name": "Esmeraldas",
+      "type": "Province"
+    },
+    {
+      "code": "EC-F",
+      "name": "Cañar",
+      "type": "Province"
+    },
+    {
+      "code": "EC-G",
+      "name": "Guayas",
+      "type": "Province"
+    },
+    {
+      "code": "EC-H",
+      "name": "Chimborazo",
+      "type": "Province"
+    },
+    {
+      "code": "EC-I",
+      "name": "Imbabura",
+      "type": "Province"
+    },
+    {
+      "code": "EC-L",
+      "name": "Loja",
+      "type": "Province"
+    },
+    {
+      "code": "EC-M",
+      "name": "Manabí",
+      "type": "Province"
+    },
+    {
+      "code": "EC-N",
+      "name": "Napo",
+      "type": "Province"
+    },
+    {
+      "code": "EC-O",
+      "name": "El Oro",
+      "type": "Province"
+    },
+    {
+      "code": "EC-P",
+      "name": "Pichincha",
+      "type": "Province"
+    },
+    {
+      "code": "EC-R",
+      "name": "Los Ríos",
+      "type": "Province"
+    },
+    {
+      "code": "EC-S",
+      "name": "Morona Santiago",
+      "type": "Province"
+    },
+    {
+      "code": "EC-SD",
+      "name": "Santo Domingo de los Tsáchilas",
+      "type": "Province"
+    },
+    {
+      "code": "EC-SE",
+      "name": "Santa Elena",
+      "type": "Province"
+    },
+    {
+      "code": "EC-T",
+      "name": "Tungurahua",
+      "type": "Province"
+    },
+    {
+      "code": "EC-U",
+      "name": "Sucumbíos",
+      "type": "Province"
+    },
+    {
+      "code": "EC-W",
+      "name": "Galápagos",
+      "type": "Province"
+    },
+    {
+      "code": "EC-X",
+      "name": "Cotopaxi",
+      "type": "Province"
+    },
+    {
+      "code": "EC-Y",
+      "name": "Pastaza",
+      "type": "Province"
+    },
+    {
+      "code": "EC-Z",
+      "name": "Zamora Chinchipe",
+      "type": "Province"
+    },
+    {
+      "code": "EE-130",
+      "name": "Alutaguse",
+      "parent": "EE-45",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-141",
+      "name": "Anija",
+      "parent": "EE-37",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-142",
+      "name": "Antsla",
+      "parent": "EE-87",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-171",
+      "name": "Elva",
+      "parent": "EE-79",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-184",
+      "name": "Haapsalu",
+      "parent": "EE-56",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-191",
+      "name": "Haljala",
+      "parent": "EE-60",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-198",
+      "name": "Harku",
+      "parent": "EE-37",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-205",
+      "name": "Hiiumaa",
+      "parent": "EE-39",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-214",
+      "name": "Häädemeeste",
+      "parent": "EE-68",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-245",
+      "name": "Jõelähtme",
+      "parent": "EE-37",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-247",
+      "name": "Jõgeva",
+      "parent": "EE-50",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-251",
+      "name": "Jõhvi",
+      "parent": "EE-45",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-255",
+      "name": "Järva",
+      "parent": "EE-52",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-272",
+      "name": "Kadrina",
+      "parent": "EE-60",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-283",
+      "name": "Kambja",
+      "parent": "EE-79",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-284",
+      "name": "Kanepi",
+      "parent": "EE-64",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-291",
+      "name": "Kastre",
+      "parent": "EE-79",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-293",
+      "name": "Kehtna",
+      "parent": "EE-71",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-296",
+      "name": "Keila",
+      "parent": "EE-37",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-303",
+      "name": "Kihnu",
+      "parent": "EE-68",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-305",
+      "name": "Kiili",
+      "parent": "EE-37",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-317",
+      "name": "Kohila",
+      "parent": "EE-71",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-321",
+      "name": "Kohtla-Järve",
+      "parent": "EE-45",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-338",
+      "name": "Kose",
+      "parent": "EE-37",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-353",
+      "name": "Kuusalu",
+      "parent": "EE-37",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-37",
+      "name": "Harjumaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-39",
+      "name": "Hiiumaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-424",
+      "name": "Loksa",
+      "parent": "EE-37",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-430",
+      "name": "Lääneranna",
+      "parent": "EE-68",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-431",
+      "name": "Lääne-Harju",
+      "parent": "EE-37",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-432",
+      "name": "Luunja",
+      "parent": "EE-79",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-441",
+      "name": "Lääne-Nigula",
+      "parent": "EE-56",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-442",
+      "name": "Lüganuse",
+      "parent": "EE-45",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-446",
+      "name": "Maardu",
+      "parent": "EE-37",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-45",
+      "name": "Ida-Virumaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-478",
+      "name": "Muhu",
+      "parent": "EE-74",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-480",
+      "name": "Mulgi",
+      "parent": "EE-84",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-486",
+      "name": "Mustvee",
+      "parent": "EE-50",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-50",
+      "name": "Jõgevamaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-503",
+      "name": "Märjamaa",
+      "parent": "EE-71",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-511",
+      "name": "Narva",
+      "parent": "EE-45",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-514",
+      "name": "Narva-Jõesuu",
+      "parent": "EE-45",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-52",
+      "name": "Järvamaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-528",
+      "name": "Nõo",
+      "parent": "EE-79",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-557",
+      "name": "Otepää",
+      "parent": "EE-81",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-56",
+      "name": "Läänemaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-567",
+      "name": "Paide",
+      "parent": "EE-52",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-586",
+      "name": "Peipsiääre",
+      "parent": "EE-79",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-60",
+      "name": "Lääne-Virumaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-615",
+      "name": "Põhja-Sakala",
+      "parent": "EE-84",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-618",
+      "name": "Põltsamaa",
+      "parent": "EE-50",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-622",
+      "name": "Põlva",
+      "parent": "EE-64",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-624",
+      "name": "Pärnu",
+      "parent": "EE-68",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-638",
+      "name": "Põhja-Pärnumaa",
+      "parent": "EE-68",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-64",
+      "name": "Põlvamaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-651",
+      "name": "Raasiku",
+      "parent": "EE-37",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-653",
+      "name": "Rae",
+      "parent": "EE-37",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-661",
+      "name": "Rakvere",
+      "parent": "EE-60",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-663",
+      "name": "Rakvere",
+      "parent": "EE-60",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-668",
+      "name": "Rapla",
+      "parent": "EE-71",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-68",
+      "name": "Pärnumaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-689",
+      "name": "Ruhnu",
+      "parent": "EE-74",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-698",
+      "name": "Rõuge",
+      "parent": "EE-87",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-708",
+      "name": "Räpina",
+      "parent": "EE-64",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-71",
+      "name": "Raplamaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-712",
+      "name": "Saarde",
+      "parent": "EE-68",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-714",
+      "name": "Saaremaa",
+      "parent": "EE-74",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-719",
+      "name": "Saku",
+      "parent": "EE-37",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-726",
+      "name": "Saue",
+      "parent": "EE-37",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-732",
+      "name": "Setomaa",
+      "parent": "EE-87",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-735",
+      "name": "Sillamäe",
+      "parent": "EE-45",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-74",
+      "name": "Saaremaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-784",
+      "name": "Tallinn",
+      "parent": "EE-37",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-79",
+      "name": "Tartumaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-792",
+      "name": "Tapa",
+      "parent": "EE-60",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-793",
+      "name": "Tartu",
+      "parent": "EE-79",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-796",
+      "name": "Tartu",
+      "parent": "EE-79",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-803",
+      "name": "Toila",
+      "parent": "EE-45",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-809",
+      "name": "Tori",
+      "parent": "EE-68",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-81",
+      "name": "Valgamaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-824",
+      "name": "Tõrva",
+      "parent": "EE-81",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-834",
+      "name": "Türi",
+      "parent": "EE-52",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-84",
+      "name": "Viljandimaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-855",
+      "name": "Valga",
+      "parent": "EE-81",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-87",
+      "name": "Võrumaa",
+      "type": "County"
+    },
+    {
+      "code": "EE-890",
+      "name": "Viimsi",
+      "parent": "EE-37",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-897",
+      "name": "Viljandi",
+      "parent": "EE-84",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-899",
+      "name": "Viljandi",
+      "parent": "EE-84",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-901",
+      "name": "Vinni",
+      "parent": "EE-60",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-903",
+      "name": "Viru-Nigula",
+      "parent": "EE-60",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-907",
+      "name": "Vormsi",
+      "parent": "EE-56",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-917",
+      "name": "Võru",
+      "parent": "EE-87",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EE-919",
+      "name": "Võru",
+      "parent": "EE-87",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "EE-928",
+      "name": "Väike-Maarja",
+      "parent": "EE-60",
+      "type": "Rural municipality"
+    },
+    {
+      "code": "EG-ALX",
+      "name": "Al Iskandarīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-ASN",
+      "name": "Aswān",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-AST",
+      "name": "Asyūţ",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-BA",
+      "name": "Al Baḩr al Aḩmar",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-BH",
+      "name": "Al Buḩayrah",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-BNS",
+      "name": "Banī Suwayf",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-C",
+      "name": "Al Qāhirah",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-DK",
+      "name": "Ad Daqahlīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-DT",
+      "name": "Dumyāţ",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-FYM",
+      "name": "Al Fayyūm",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-GH",
+      "name": "Al Gharbīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-GZ",
+      "name": "Al Jīzah",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-IS",
+      "name": "Al Ismā'īlīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-JS",
+      "name": "Janūb Sīnā'",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-KB",
+      "name": "Al Qalyūbīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-KFS",
+      "name": "Kafr ash Shaykh",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-KN",
+      "name": "Qinā",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-LX",
+      "name": "Al Uqşur",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-MN",
+      "name": "Al Minyā",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-MNF",
+      "name": "Al Minūfīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-MT",
+      "name": "Maţrūḩ",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-PTS",
+      "name": "Būr Sa‘īd",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-SHG",
+      "name": "Sūhāj",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-SHR",
+      "name": "Ash Sharqīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-SIN",
+      "name": "Shamāl Sīnā'",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-SUZ",
+      "name": "As Suways",
+      "type": "Governorate"
+    },
+    {
+      "code": "EG-WAD",
+      "name": "Al Wādī al Jadīd",
+      "type": "Governorate"
+    },
+    {
+      "code": "ER-AN",
+      "name": "Ansabā",
+      "type": "Region"
+    },
+    {
+      "code": "ER-DK",
+      "name": "Debubawi K’eyyĭḥ Baḥri",
+      "type": "Region"
+    },
+    {
+      "code": "ER-DU",
+      "name": "Al Janūbī",
+      "type": "Region"
+    },
+    {
+      "code": "ER-GB",
+      "name": "Gash-Barka",
+      "type": "Region"
+    },
+    {
+      "code": "ER-MA",
+      "name": "Al Awsaţ",
+      "type": "Region"
+    },
+    {
+      "code": "ER-SK",
+      "name": "Semienawi K’eyyĭḥ Baḥri",
+      "type": "Region"
+    },
+    {
+      "code": "ES-A",
+      "name": "Alacant*",
+      "parent": "ES-VC",
+      "type": "Province"
+    },
+    {
+      "code": "ES-AB",
+      "name": "Albacete",
+      "parent": "ES-CM",
+      "type": "Province"
+    },
+    {
+      "code": "ES-AL",
+      "name": "Almería",
+      "parent": "ES-AN",
+      "type": "Province"
+    },
+    {
+      "code": "ES-AN",
+      "name": "Andalucía",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-AR",
+      "name": "Aragón",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-AS",
+      "name": "Asturias, Principado de",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-AV",
+      "name": "Ávila",
+      "parent": "ES-CL",
+      "type": "Province"
+    },
+    {
+      "code": "ES-B",
+      "name": "Barcelona [Barcelona]",
+      "parent": "ES-CT",
+      "type": "Province"
+    },
+    {
+      "code": "ES-BA",
+      "name": "Badajoz",
+      "parent": "ES-EX",
+      "type": "Province"
+    },
+    {
+      "code": "ES-BI",
+      "name": "Bizkaia",
+      "parent": "ES-PV",
+      "type": "Province"
+    },
+    {
+      "code": "ES-BU",
+      "name": "Burgos",
+      "parent": "ES-CL",
+      "type": "Province"
+    },
+    {
+      "code": "ES-C",
+      "name": "A Coruña [La Coruña]",
+      "parent": "ES-GA",
+      "type": "Province"
+    },
+    {
+      "code": "ES-CA",
+      "name": "Cádiz",
+      "parent": "ES-AN",
+      "type": "Province"
+    },
+    {
+      "code": "ES-CB",
+      "name": "Cantabria",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-CC",
+      "name": "Cáceres",
+      "parent": "ES-EX",
+      "type": "Province"
+    },
+    {
+      "code": "ES-CE",
+      "name": "Ceuta",
+      "type": "Autonomous city in north africa"
+    },
+    {
+      "code": "ES-CL",
+      "name": "Castilla y León",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-CM",
+      "name": "Castilla-La Mancha",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-CN",
+      "name": "Canarias",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-CO",
+      "name": "Córdoba",
+      "parent": "ES-AN",
+      "type": "Province"
+    },
+    {
+      "code": "ES-CR",
+      "name": "Ciudad Real",
+      "parent": "ES-CM",
+      "type": "Province"
+    },
+    {
+      "code": "ES-CS",
+      "name": "Castelló*",
+      "parent": "ES-VC",
+      "type": "Province"
+    },
+    {
+      "code": "ES-CT",
+      "name": "Catalunya [Cataluña]",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-CU",
+      "name": "Cuenca",
+      "parent": "ES-CM",
+      "type": "Province"
+    },
+    {
+      "code": "ES-EX",
+      "name": "Extremadura",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-GA",
+      "name": "Galicia [Galicia]",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-GC",
+      "name": "Las Palmas",
+      "parent": "ES-CN",
+      "type": "Province"
+    },
+    {
+      "code": "ES-GI",
+      "name": "Girona [Gerona]",
+      "parent": "ES-CT",
+      "type": "Province"
+    },
+    {
+      "code": "ES-GR",
+      "name": "Granada",
+      "parent": "ES-AN",
+      "type": "Province"
+    },
+    {
+      "code": "ES-GU",
+      "name": "Guadalajara",
+      "parent": "ES-CM",
+      "type": "Province"
+    },
+    {
+      "code": "ES-H",
+      "name": "Huelva",
+      "parent": "ES-AN",
+      "type": "Province"
+    },
+    {
+      "code": "ES-HU",
+      "name": "Huesca",
+      "parent": "ES-AR",
+      "type": "Province"
+    },
+    {
+      "code": "ES-IB",
+      "name": "Illes Balears [Islas Baleares]",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-J",
+      "name": "Jaén",
+      "parent": "ES-AN",
+      "type": "Province"
+    },
+    {
+      "code": "ES-L",
+      "name": "Lleida [Lérida]",
+      "parent": "ES-CT",
+      "type": "Province"
+    },
+    {
+      "code": "ES-LE",
+      "name": "León",
+      "parent": "ES-CL",
+      "type": "Province"
+    },
+    {
+      "code": "ES-LO",
+      "name": "La Rioja",
+      "parent": "ES-RI",
+      "type": "Province"
+    },
+    {
+      "code": "ES-LU",
+      "name": "Lugo [Lugo]",
+      "parent": "ES-GA",
+      "type": "Province"
+    },
+    {
+      "code": "ES-M",
+      "name": "Madrid",
+      "parent": "ES-MD",
+      "type": "Province"
+    },
+    {
+      "code": "ES-MA",
+      "name": "Málaga",
+      "parent": "ES-AN",
+      "type": "Province"
+    },
+    {
+      "code": "ES-MC",
+      "name": "Murcia, Región de",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-MD",
+      "name": "Madrid, Comunidad de",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-ML",
+      "name": "Melilla",
+      "type": "Autonomous city in north africa"
+    },
+    {
+      "code": "ES-MU",
+      "name": "Murcia",
+      "parent": "ES-MC",
+      "type": "Province"
+    },
+    {
+      "code": "ES-NA",
+      "name": "Nafarroa*",
+      "parent": "ES-NC",
+      "type": "Province"
+    },
+    {
+      "code": "ES-NC",
+      "name": "Nafarroako Foru Komunitatea*",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-O",
+      "name": "Asturias",
+      "parent": "ES-AS",
+      "type": "Province"
+    },
+    {
+      "code": "ES-OR",
+      "name": "Ourense [Orense]",
+      "parent": "ES-GA",
+      "type": "Province"
+    },
+    {
+      "code": "ES-P",
+      "name": "Palencia",
+      "parent": "ES-CL",
+      "type": "Province"
+    },
+    {
+      "code": "ES-PM",
+      "name": "Illes Balears [Islas Baleares]",
+      "parent": "ES-IB",
+      "type": "Province"
+    },
+    {
+      "code": "ES-PO",
+      "name": "Pontevedra [Pontevedra]",
+      "parent": "ES-GA",
+      "type": "Province"
+    },
+    {
+      "code": "ES-PV",
+      "name": "Euskal Herria",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-RI",
+      "name": "La Rioja",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-S",
+      "name": "Cantabria",
+      "parent": "ES-CB",
+      "type": "Province"
+    },
+    {
+      "code": "ES-SA",
+      "name": "Salamanca",
+      "parent": "ES-CL",
+      "type": "Province"
+    },
+    {
+      "code": "ES-SE",
+      "name": "Sevilla",
+      "parent": "ES-AN",
+      "type": "Province"
+    },
+    {
+      "code": "ES-SG",
+      "name": "Segovia",
+      "parent": "ES-CL",
+      "type": "Province"
+    },
+    {
+      "code": "ES-SO",
+      "name": "Soria",
+      "parent": "ES-CL",
+      "type": "Province"
+    },
+    {
+      "code": "ES-SS",
+      "name": "Gipuzkoa",
+      "parent": "ES-PV",
+      "type": "Province"
+    },
+    {
+      "code": "ES-T",
+      "name": "Tarragona [Tarragona]",
+      "parent": "ES-CT",
+      "type": "Province"
+    },
+    {
+      "code": "ES-TE",
+      "name": "Teruel",
+      "parent": "ES-AR",
+      "type": "Province"
+    },
+    {
+      "code": "ES-TF",
+      "name": "Santa Cruz de Tenerife",
+      "parent": "ES-CN",
+      "type": "Province"
+    },
+    {
+      "code": "ES-TO",
+      "name": "Toledo",
+      "parent": "ES-CM",
+      "type": "Province"
+    },
+    {
+      "code": "ES-V",
+      "name": "Valencia",
+      "parent": "ES-VC",
+      "type": "Province"
+    },
+    {
+      "code": "ES-VA",
+      "name": "Valladolid",
+      "parent": "ES-CL",
+      "type": "Province"
+    },
+    {
+      "code": "ES-VC",
+      "name": "Valenciana, Comunidad",
+      "type": "Autonomous community"
+    },
+    {
+      "code": "ES-VI",
+      "name": "Araba*",
+      "parent": "ES-PV",
+      "type": "Province"
+    },
+    {
+      "code": "ES-Z",
+      "name": "Zaragoza",
+      "parent": "ES-AR",
+      "type": "Province"
+    },
+    {
+      "code": "ES-ZA",
+      "name": "Zamora",
+      "parent": "ES-CL",
+      "type": "Province"
+    },
+    {
+      "code": "ET-AA",
+      "name": "Addis Ababa",
+      "type": "Administration"
+    },
+    {
+      "code": "ET-AF",
+      "name": "Afar",
+      "type": "Regional state"
+    },
+    {
+      "code": "ET-AM",
+      "name": "Amara",
+      "type": "Regional state"
+    },
+    {
+      "code": "ET-BE",
+      "name": "Benshangul-Gumaz",
+      "type": "Regional state"
+    },
+    {
+      "code": "ET-DD",
+      "name": "Dire Dawa",
+      "type": "Administration"
+    },
+    {
+      "code": "ET-GA",
+      "name": "Gambela Peoples",
+      "type": "Regional state"
+    },
+    {
+      "code": "ET-HA",
+      "name": "Harari People",
+      "type": "Regional state"
+    },
+    {
+      "code": "ET-OR",
+      "name": "Oromia",
+      "type": "Regional state"
+    },
+    {
+      "code": "ET-SI",
+      "name": "Sidama",
+      "type": "Regional state"
+    },
+    {
+      "code": "ET-SN",
+      "name": "Southern Nations, Nationalities and Peoples",
+      "type": "Regional state"
+    },
+    {
+      "code": "ET-SO",
+      "name": "Somali",
+      "type": "Regional state"
+    },
+    {
+      "code": "ET-SW",
+      "name": "Southwest Ethiopia Peoples",
+      "type": "Regional state"
+    },
+    {
+      "code": "ET-TI",
+      "name": "Tigrai",
+      "type": "Regional state"
+    },
+    {
+      "code": "FI-01",
+      "name": "Landskapet Åland",
+      "type": "Region"
+    },
+    {
+      "code": "FI-02",
+      "name": "Etelä-Karjala",
+      "type": "Region"
+    },
+    {
+      "code": "FI-03",
+      "name": "Etelä-Pohjanmaa",
+      "type": "Region"
+    },
+    {
+      "code": "FI-04",
+      "name": "Etelä-Savo",
+      "type": "Region"
+    },
+    {
+      "code": "FI-05",
+      "name": "Kainuu",
+      "type": "Region"
+    },
+    {
+      "code": "FI-06",
+      "name": "Kanta-Häme",
+      "type": "Region"
+    },
+    {
+      "code": "FI-07",
+      "name": "Keski-Pohjanmaa",
+      "type": "Region"
+    },
+    {
+      "code": "FI-08",
+      "name": "Keski-Suomi",
+      "type": "Region"
+    },
+    {
+      "code": "FI-09",
+      "name": "Kymenlaakso",
+      "type": "Region"
+    },
+    {
+      "code": "FI-10",
+      "name": "Lappi",
+      "type": "Region"
+    },
+    {
+      "code": "FI-11",
+      "name": "Pirkanmaa",
+      "type": "Region"
+    },
+    {
+      "code": "FI-12",
+      "name": "Pohjanmaa",
+      "type": "Region"
+    },
+    {
+      "code": "FI-13",
+      "name": "Pohjois-Karjala",
+      "type": "Region"
+    },
+    {
+      "code": "FI-14",
+      "name": "Pohjois-Pohjanmaa",
+      "type": "Region"
+    },
+    {
+      "code": "FI-15",
+      "name": "Pohjois-Savo",
+      "type": "Region"
+    },
+    {
+      "code": "FI-16",
+      "name": "Päijät-Häme",
+      "type": "Region"
+    },
+    {
+      "code": "FI-17",
+      "name": "Satakunta",
+      "type": "Region"
+    },
+    {
+      "code": "FI-18",
+      "name": "Uusimaa",
+      "type": "Region"
+    },
+    {
+      "code": "FI-19",
+      "name": "Varsinais-Suomi",
+      "type": "Region"
+    },
+    {
+      "code": "FJ-01",
+      "name": "Ba",
+      "parent": "FJ-W",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-02",
+      "name": "Bua",
+      "parent": "FJ-N",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-03",
+      "name": "Cakaudrove",
+      "parent": "FJ-N",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-04",
+      "name": "Kadavu",
+      "parent": "FJ-E",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-05",
+      "name": "Lau",
+      "parent": "FJ-E",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-06",
+      "name": "Lomaiviti",
+      "parent": "FJ-E",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-07",
+      "name": "Macuata",
+      "parent": "FJ-N",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-08",
+      "name": "Nadroga and Navosa",
+      "parent": "FJ-W",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-09",
+      "name": "Naitasiri",
+      "parent": "FJ-C",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-10",
+      "name": "Namosi",
+      "parent": "FJ-C",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-11",
+      "name": "Ra",
+      "parent": "FJ-W",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-12",
+      "name": "Rewa",
+      "parent": "FJ-C",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-13",
+      "name": "Serua",
+      "parent": "FJ-C",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-14",
+      "name": "Tailevu",
+      "parent": "FJ-C",
+      "type": "Province"
+    },
+    {
+      "code": "FJ-C",
+      "name": "Central",
+      "type": "Division"
+    },
+    {
+      "code": "FJ-E",
+      "name": "Eastern",
+      "type": "Division"
+    },
+    {
+      "code": "FJ-N",
+      "name": "Northern",
+      "type": "Division"
+    },
+    {
+      "code": "FJ-R",
+      "name": "Rotuma",
+      "type": "Dependency"
+    },
+    {
+      "code": "FJ-W",
+      "name": "Western",
+      "type": "Division"
+    },
+    {
+      "code": "FM-KSA",
+      "name": "Kosrae",
+      "type": "State"
+    },
+    {
+      "code": "FM-PNI",
+      "name": "Pohnpei",
+      "type": "State"
+    },
+    {
+      "code": "FM-TRK",
+      "name": "Chuuk",
+      "type": "State"
+    },
+    {
+      "code": "FM-YAP",
+      "name": "Yap",
+      "type": "State"
+    },
+    {
+      "code": "FR-01",
+      "name": "Ain",
+      "parent": "FR-ARA",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-02",
+      "name": "Aisne",
+      "parent": "FR-HDF",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-03",
+      "name": "Allier",
+      "parent": "FR-ARA",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-04",
+      "name": "Alpes-de-Haute-Provence",
+      "parent": "FR-PAC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-05",
+      "name": "Hautes-Alpes",
+      "parent": "FR-PAC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-06",
+      "name": "Alpes-Maritimes",
+      "parent": "FR-PAC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-07",
+      "name": "Ardèche",
+      "parent": "FR-ARA",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-08",
+      "name": "Ardennes",
+      "parent": "FR-GES",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-09",
+      "name": "Ariège",
+      "parent": "FR-OCC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-10",
+      "name": "Aube",
+      "parent": "FR-GES",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-11",
+      "name": "Aude",
+      "parent": "FR-OCC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-12",
+      "name": "Aveyron",
+      "parent": "FR-OCC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-13",
+      "name": "Bouches-du-Rhône",
+      "parent": "FR-PAC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-14",
+      "name": "Calvados",
+      "parent": "FR-NOR",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-15",
+      "name": "Cantal",
+      "parent": "FR-ARA",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-16",
+      "name": "Charente",
+      "parent": "FR-NAQ",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-17",
+      "name": "Charente-Maritime",
+      "parent": "FR-NAQ",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-18",
+      "name": "Cher",
+      "parent": "FR-CVL",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-19",
+      "name": "Corrèze",
+      "parent": "FR-NAQ",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-20R",
+      "name": "Corse",
+      "type": "Metropolitan collectivity with special status"
+    },
+    {
+      "code": "FR-21",
+      "name": "Côte-d'Or",
+      "parent": "FR-BFC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-22",
+      "name": "Côtes-d'Armor",
+      "parent": "FR-BRE",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-23",
+      "name": "Creuse",
+      "parent": "FR-NAQ",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-24",
+      "name": "Dordogne",
+      "parent": "FR-NAQ",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-25",
+      "name": "Doubs",
+      "parent": "FR-BFC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-26",
+      "name": "Drôme",
+      "parent": "FR-ARA",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-27",
+      "name": "Eure",
+      "parent": "FR-NOR",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-28",
+      "name": "Eure-et-Loir",
+      "parent": "FR-CVL",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-29",
+      "name": "Finistère",
+      "parent": "FR-BRE",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-2A",
+      "name": "Corse-du-Sud",
+      "parent": "FR-20R",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-2B",
+      "name": "Haute-Corse",
+      "parent": "FR-20R",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-30",
+      "name": "Gard",
+      "parent": "FR-OCC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-31",
+      "name": "Haute-Garonne",
+      "parent": "FR-OCC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-32",
+      "name": "Gers",
+      "parent": "FR-OCC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-33",
+      "name": "Gironde",
+      "parent": "FR-NAQ",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-34",
+      "name": "Hérault",
+      "parent": "FR-OCC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-35",
+      "name": "Ille-et-Vilaine",
+      "parent": "FR-BRE",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-36",
+      "name": "Indre",
+      "parent": "FR-CVL",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-37",
+      "name": "Indre-et-Loire",
+      "parent": "FR-CVL",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-38",
+      "name": "Isère",
+      "parent": "FR-ARA",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-39",
+      "name": "Jura",
+      "parent": "FR-BFC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-40",
+      "name": "Landes",
+      "parent": "FR-NAQ",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-41",
+      "name": "Loir-et-Cher",
+      "parent": "FR-CVL",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-42",
+      "name": "Loire",
+      "parent": "FR-ARA",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-43",
+      "name": "Haute-Loire",
+      "parent": "FR-ARA",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-44",
+      "name": "Loire-Atlantique",
+      "parent": "FR-PDL",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-45",
+      "name": "Loiret",
+      "parent": "FR-CVL",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-46",
+      "name": "Lot",
+      "parent": "FR-OCC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-47",
+      "name": "Lot-et-Garonne",
+      "parent": "FR-NAQ",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-48",
+      "name": "Lozère",
+      "parent": "FR-OCC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-49",
+      "name": "Maine-et-Loire",
+      "parent": "FR-PDL",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-50",
+      "name": "Manche",
+      "parent": "FR-NOR",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-51",
+      "name": "Marne",
+      "parent": "FR-GES",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-52",
+      "name": "Haute-Marne",
+      "parent": "FR-GES",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-53",
+      "name": "Mayenne",
+      "parent": "FR-PDL",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-54",
+      "name": "Meurthe-et-Moselle",
+      "parent": "FR-GES",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-55",
+      "name": "Meuse",
+      "parent": "FR-GES",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-56",
+      "name": "Morbihan",
+      "parent": "FR-BRE",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-57",
+      "name": "Moselle",
+      "parent": "FR-GES",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-58",
+      "name": "Nièvre",
+      "parent": "FR-BFC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-59",
+      "name": "Nord",
+      "parent": "FR-HDF",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-60",
+      "name": "Oise",
+      "parent": "FR-HDF",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-61",
+      "name": "Orne",
+      "parent": "FR-NOR",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-62",
+      "name": "Pas-de-Calais",
+      "parent": "FR-HDF",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-63",
+      "name": "Puy-de-Dôme",
+      "parent": "FR-ARA",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-64",
+      "name": "Pyrénées-Atlantiques",
+      "parent": "FR-NAQ",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-65",
+      "name": "Hautes-Pyrénées",
+      "parent": "FR-OCC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-66",
+      "name": "Pyrénées-Orientales",
+      "parent": "FR-OCC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-67",
+      "name": "Bas-Rhin",
+      "parent": "FR-6AE",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-68",
+      "name": "Haut-Rhin",
+      "parent": "FR-6AE",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-69",
+      "name": "Rhône",
+      "parent": "FR-ARA",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-69M",
+      "name": "Métropole de Lyon",
+      "parent": "FR-ARA",
+      "type": "Metropolitan collectivity with special status"
+    },
+    {
+      "code": "FR-6AE",
+      "name": "Alsace",
+      "parent": "FR-GES",
+      "type": "European collectivity"
+    },
+    {
+      "code": "FR-70",
+      "name": "Haute-Saône",
+      "parent": "FR-BFC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-71",
+      "name": "Saône-et-Loire",
+      "parent": "FR-BFC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-72",
+      "name": "Sarthe",
+      "parent": "FR-PDL",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-73",
+      "name": "Savoie",
+      "parent": "FR-ARA",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-74",
+      "name": "Haute-Savoie",
+      "parent": "FR-ARA",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-75C",
+      "name": "Paris",
+      "parent": "FR-IDF",
+      "type": "Metropolitan collectivity with special status"
+    },
+    {
+      "code": "FR-76",
+      "name": "Seine-Maritime",
+      "parent": "FR-NOR",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-77",
+      "name": "Seine-et-Marne",
+      "parent": "FR-IDF",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-78",
+      "name": "Yvelines",
+      "parent": "FR-IDF",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-79",
+      "name": "Deux-Sèvres",
+      "parent": "FR-NAQ",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-80",
+      "name": "Somme",
+      "parent": "FR-HDF",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-81",
+      "name": "Tarn",
+      "parent": "FR-OCC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-82",
+      "name": "Tarn-et-Garonne",
+      "parent": "FR-OCC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-83",
+      "name": "Var",
+      "parent": "FR-PAC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-84",
+      "name": "Vaucluse",
+      "parent": "FR-PAC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-85",
+      "name": "Vendée",
+      "parent": "FR-PDL",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-86",
+      "name": "Vienne",
+      "parent": "FR-NAQ",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-87",
+      "name": "Haute-Vienne",
+      "parent": "FR-NAQ",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-88",
+      "name": "Vosges",
+      "parent": "FR-GES",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-89",
+      "name": "Yonne",
+      "parent": "FR-BFC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-90",
+      "name": "Territoire de Belfort",
+      "parent": "FR-BFC",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-91",
+      "name": "Essonne",
+      "parent": "FR-IDF",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-92",
+      "name": "Hauts-de-Seine",
+      "parent": "FR-IDF",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-93",
+      "name": "Seine-Saint-Denis",
+      "parent": "FR-IDF",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-94",
+      "name": "Val-de-Marne",
+      "parent": "FR-IDF",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-95",
+      "name": "Val-d'Oise",
+      "parent": "FR-IDF",
+      "type": "Metropolitan department"
+    },
+    {
+      "code": "FR-971",
+      "name": "Guadeloupe",
+      "type": "Overseas departmental collectivity"
+    },
+    {
+      "code": "FR-972",
+      "name": "Martinique",
+      "type": "Overseas unique territorial collectivity"
+    },
+    {
+      "code": "FR-973",
+      "name": "Guyane (française)",
+      "type": "Overseas unique territorial collectivity"
+    },
+    {
+      "code": "FR-974",
+      "name": "La Réunion",
+      "type": "Overseas departmental collectivity"
+    },
+    {
+      "code": "FR-976",
+      "name": "Mayotte",
+      "type": "Overseas departmental collectivity"
+    },
+    {
+      "code": "FR-ARA",
+      "name": "Auvergne-Rhône-Alpes",
+      "type": "Metropolitan region"
+    },
+    {
+      "code": "FR-BFC",
+      "name": "Bourgogne-Franche-Comté",
+      "type": "Metropolitan region"
+    },
+    {
+      "code": "FR-BL",
+      "name": "Saint-Barthélemy",
+      "type": "Overseas collectivity"
+    },
+    {
+      "code": "FR-BRE",
+      "name": "Bretagne",
+      "type": "Metropolitan region"
+    },
+    {
+      "code": "FR-CP",
+      "name": "Clipperton",
+      "type": "Dependency"
+    },
+    {
+      "code": "FR-CVL",
+      "name": "Centre-Val de Loire",
+      "type": "Metropolitan region"
+    },
+    {
+      "code": "FR-GES",
+      "name": "Grand-Est",
+      "type": "Metropolitan region"
+    },
+    {
+      "code": "FR-HDF",
+      "name": "Hauts-de-France",
+      "type": "Metropolitan region"
+    },
+    {
+      "code": "FR-IDF",
+      "name": "Île-de-France",
+      "type": "Metropolitan region"
+    },
+    {
+      "code": "FR-MF",
+      "name": "Saint-Martin",
+      "type": "Overseas collectivity"
+    },
+    {
+      "code": "FR-NAQ",
+      "name": "Nouvelle-Aquitaine",
+      "type": "Metropolitan region"
+    },
+    {
+      "code": "FR-NC",
+      "name": "Nouvelle-Calédonie",
+      "type": "Overseas collectivity with special status"
+    },
+    {
+      "code": "FR-NOR",
+      "name": "Normandie",
+      "type": "Metropolitan region"
+    },
+    {
+      "code": "FR-OCC",
+      "name": "Occitanie",
+      "type": "Metropolitan region"
+    },
+    {
+      "code": "FR-PAC",
+      "name": "Provence-Alpes-Côte-d’Azur",
+      "type": "Metropolitan region"
+    },
+    {
+      "code": "FR-PDL",
+      "name": "Pays-de-la-Loire",
+      "type": "Metropolitan region"
+    },
+    {
+      "code": "FR-PF",
+      "name": "Polynésie française",
+      "type": "Overseas collectivity"
+    },
+    {
+      "code": "FR-PM",
+      "name": "Saint-Pierre-et-Miquelon",
+      "type": "Overseas collectivity"
+    },
+    {
+      "code": "FR-TF",
+      "name": "Terres australes françaises",
+      "type": "Overseas territory"
+    },
+    {
+      "code": "FR-WF",
+      "name": "Wallis-et-Futuna",
+      "type": "Overseas collectivity"
+    },
+    {
+      "code": "GA-1",
+      "name": "Estuaire",
+      "type": "Province"
+    },
+    {
+      "code": "GA-2",
+      "name": "Haut-Ogooué",
+      "type": "Province"
+    },
+    {
+      "code": "GA-3",
+      "name": "Moyen-Ogooué",
+      "type": "Province"
+    },
+    {
+      "code": "GA-4",
+      "name": "Ngounié",
+      "type": "Province"
+    },
+    {
+      "code": "GA-5",
+      "name": "Nyanga",
+      "type": "Province"
+    },
+    {
+      "code": "GA-6",
+      "name": "Ogooué-Ivindo",
+      "type": "Province"
+    },
+    {
+      "code": "GA-7",
+      "name": "Ogooué-Lolo",
+      "type": "Province"
+    },
+    {
+      "code": "GA-8",
+      "name": "Ogooué-Maritime",
+      "type": "Province"
+    },
+    {
+      "code": "GA-9",
+      "name": "Woleu-Ntem",
+      "type": "Province"
+    },
+    {
+      "code": "GB-ABC",
+      "name": "Armagh City, Banbridge and Craigavon",
+      "parent": "GB-NIR",
+      "type": "District"
+    },
+    {
+      "code": "GB-ABD",
+      "name": "Aberdeenshire",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-ABE",
+      "name": "Aberdeen City",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-AGB",
+      "name": "Argyll and Bute",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-AGY",
+      "name": "Isle of Anglesey [Sir Ynys Môn GB-YNM]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-AND",
+      "name": "Ards and North Down",
+      "parent": "GB-NIR",
+      "type": "District"
+    },
+    {
+      "code": "GB-ANN",
+      "name": "Antrim and Newtownabbey",
+      "parent": "GB-NIR",
+      "type": "District"
+    },
+    {
+      "code": "GB-ANS",
+      "name": "Angus",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-BAS",
+      "name": "Bath and North East Somerset",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-BBD",
+      "name": "Blackburn with Darwen",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-BCP",
+      "name": "Bournemouth, Christchurch and Poole",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-BDF",
+      "name": "Bedford",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-BDG",
+      "name": "Barking and Dagenham",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-BEN",
+      "name": "Brent",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-BEX",
+      "name": "Bexley",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-BFS",
+      "name": "Belfast City",
+      "parent": "GB-NIR",
+      "type": "District"
+    },
+    {
+      "code": "GB-BGE",
+      "name": "Bridgend [Pen-y-bont ar Ogwr GB-POG]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-BGW",
+      "name": "Blaenau Gwent",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-BIR",
+      "name": "Birmingham",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-BKM",
+      "name": "Buckinghamshire",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-BNE",
+      "name": "Barnet",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-BNH",
+      "name": "Brighton and Hove",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-BNS",
+      "name": "Barnsley",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-BOL",
+      "name": "Bolton",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-BPL",
+      "name": "Blackpool",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-BRC",
+      "name": "Bracknell Forest",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-BRD",
+      "name": "Bradford",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-BRY",
+      "name": "Bromley",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-BST",
+      "name": "Bristol, City of",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-BUR",
+      "name": "Bury",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-CAM",
+      "name": "Cambridgeshire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-CAY",
+      "name": "Caerphilly [Caerffili GB-CAF]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-CBF",
+      "name": "Central Bedfordshire",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-CCG",
+      "name": "Causeway Coast and Glens",
+      "parent": "GB-NIR",
+      "type": "District"
+    },
+    {
+      "code": "GB-CGN",
+      "name": "Ceredigion [Sir Ceredigion]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-CHE",
+      "name": "Cheshire East",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-CHW",
+      "name": "Cheshire West and Chester",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-CLD",
+      "name": "Calderdale",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-CLK",
+      "name": "Clackmannanshire",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-CMA",
+      "name": "Cumbria",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-CMD",
+      "name": "Camden",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-CMN",
+      "name": "Carmarthenshire [Sir Gaerfyrddin GB-GFY]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-CON",
+      "name": "Cornwall",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-COV",
+      "name": "Coventry",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-CRF",
+      "name": "Cardiff [Caerdydd GB-CRD]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-CRY",
+      "name": "Croydon",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-CWY",
+      "name": "Conwy",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-DAL",
+      "name": "Darlington",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-DBY",
+      "name": "Derbyshire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-DEN",
+      "name": "Denbighshire [Sir Ddinbych GB-DDB]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-DER",
+      "name": "Derby",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-DEV",
+      "name": "Devon",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-DGY",
+      "name": "Dumfries and Galloway",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-DNC",
+      "name": "Doncaster",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-DND",
+      "name": "Dundee City",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-DOR",
+      "name": "Dorset",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-DRS",
+      "name": "Derry and Strabane",
+      "parent": "GB-NIR",
+      "type": "District"
+    },
+    {
+      "code": "GB-DUD",
+      "name": "Dudley",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-DUR",
+      "name": "Durham, County",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-EAL",
+      "name": "Ealing",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-EAY",
+      "name": "East Ayrshire",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-EDH",
+      "name": "Edinburgh, City of",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-EDU",
+      "name": "East Dunbartonshire",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-ELN",
+      "name": "East Lothian",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-ELS",
+      "name": "Eilean Siar",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-ENF",
+      "name": "Enfield",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-ENG",
+      "name": "England",
+      "type": "Country"
+    },
+    {
+      "code": "GB-ERW",
+      "name": "East Renfrewshire",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-ERY",
+      "name": "East Riding of Yorkshire",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-ESS",
+      "name": "Essex",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-ESX",
+      "name": "East Sussex",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-FAL",
+      "name": "Falkirk",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-FIF",
+      "name": "Fife",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-FLN",
+      "name": "Flintshire [Sir y Fflint GB-FFL]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-FMO",
+      "name": "Fermanagh and Omagh",
+      "parent": "GB-NIR",
+      "type": "District"
+    },
+    {
+      "code": "GB-GAT",
+      "name": "Gateshead",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-GLG",
+      "name": "Glasgow City",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-GLS",
+      "name": "Gloucestershire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-GRE",
+      "name": "Greenwich",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-GWN",
+      "name": "Gwynedd",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-HAL",
+      "name": "Halton",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-HAM",
+      "name": "Hampshire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-HAV",
+      "name": "Havering",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-HCK",
+      "name": "Hackney",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-HEF",
+      "name": "Herefordshire",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-HIL",
+      "name": "Hillingdon",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-HLD",
+      "name": "Highland",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-HMF",
+      "name": "Hammersmith and Fulham",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-HNS",
+      "name": "Hounslow",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-HPL",
+      "name": "Hartlepool",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-HRT",
+      "name": "Hertfordshire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-HRW",
+      "name": "Harrow",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-HRY",
+      "name": "Haringey",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-IOS",
+      "name": "Isles of Scilly",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-IOW",
+      "name": "Isle of Wight",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-ISL",
+      "name": "Islington",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-IVC",
+      "name": "Inverclyde",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-KEC",
+      "name": "Kensington and Chelsea",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-KEN",
+      "name": "Kent",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-KHL",
+      "name": "Kingston upon Hull",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-KIR",
+      "name": "Kirklees",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-KTT",
+      "name": "Kingston upon Thames",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-KWL",
+      "name": "Knowsley",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-LAN",
+      "name": "Lancashire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-LBC",
+      "name": "Lisburn and Castlereagh",
+      "parent": "GB-NIR",
+      "type": "District"
+    },
+    {
+      "code": "GB-LBH",
+      "name": "Lambeth",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-LCE",
+      "name": "Leicester",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-LDS",
+      "name": "Leeds",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-LEC",
+      "name": "Leicestershire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-LEW",
+      "name": "Lewisham",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-LIN",
+      "name": "Lincolnshire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-LIV",
+      "name": "Liverpool",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-LND",
+      "name": "London, City of",
+      "parent": "GB-ENG",
+      "type": "City corporation"
+    },
+    {
+      "code": "GB-LUT",
+      "name": "Luton",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-MAN",
+      "name": "Manchester",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-MDB",
+      "name": "Middlesbrough",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-MDW",
+      "name": "Medway",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-MEA",
+      "name": "Mid and East Antrim",
+      "parent": "GB-NIR",
+      "type": "District"
+    },
+    {
+      "code": "GB-MIK",
+      "name": "Milton Keynes",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-MLN",
+      "name": "Midlothian",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-MON",
+      "name": "Monmouthshire [Sir Fynwy GB-FYN]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-MRT",
+      "name": "Merton",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-MRY",
+      "name": "Moray",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-MTY",
+      "name": "Merthyr Tydfil [Merthyr Tudful GB-MTU]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-MUL",
+      "name": "Mid-Ulster",
+      "parent": "GB-NIR",
+      "type": "District"
+    },
+    {
+      "code": "GB-NAY",
+      "name": "North Ayrshire",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-NBL",
+      "name": "Northumberland",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-NEL",
+      "name": "North East Lincolnshire",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-NET",
+      "name": "Newcastle upon Tyne",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-NFK",
+      "name": "Norfolk",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-NGM",
+      "name": "Nottingham",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-NIR",
+      "name": "Northern Ireland",
+      "type": "Province"
+    },
+    {
+      "code": "GB-NLK",
+      "name": "North Lanarkshire",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-NLN",
+      "name": "North Lincolnshire",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-NMD",
+      "name": "Newry, Mourne and Down",
+      "parent": "GB-NIR",
+      "type": "District"
+    },
+    {
+      "code": "GB-NNH",
+      "name": "North Northamptonshire",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-NSM",
+      "name": "North Somerset",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-NTL",
+      "name": "Neath Port Talbot [Castell-nedd Port Talbot GB-CTL]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-NTT",
+      "name": "Nottinghamshire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-NTY",
+      "name": "North Tyneside",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-NWM",
+      "name": "Newham",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-NWP",
+      "name": "Newport [Casnewydd GB-CNW]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-NYK",
+      "name": "North Yorkshire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-OLD",
+      "name": "Oldham",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-ORK",
+      "name": "Orkney Islands",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-OXF",
+      "name": "Oxfordshire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-PEM",
+      "name": "Pembrokeshire [Sir Benfro GB-BNF]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-PKN",
+      "name": "Perth and Kinross",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-PLY",
+      "name": "Plymouth",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-POR",
+      "name": "Portsmouth",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-POW",
+      "name": "Powys",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-PTE",
+      "name": "Peterborough",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-RCC",
+      "name": "Redcar and Cleveland",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-RCH",
+      "name": "Rochdale",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-RCT",
+      "name": "Rhondda Cynon Taff [Rhondda CynonTaf]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-RDB",
+      "name": "Redbridge",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-RDG",
+      "name": "Reading",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-RFW",
+      "name": "Renfrewshire",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-RIC",
+      "name": "Richmond upon Thames",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-ROT",
+      "name": "Rotherham",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-RUT",
+      "name": "Rutland",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-SAW",
+      "name": "Sandwell",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-SAY",
+      "name": "South Ayrshire",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-SCB",
+      "name": "Scottish Borders",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-SCT",
+      "name": "Scotland",
+      "type": "Country"
+    },
+    {
+      "code": "GB-SFK",
+      "name": "Suffolk",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-SFT",
+      "name": "Sefton",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-SGC",
+      "name": "South Gloucestershire",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-SHF",
+      "name": "Sheffield",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-SHN",
+      "name": "St. Helens",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-SHR",
+      "name": "Shropshire",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-SKP",
+      "name": "Stockport",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-SLF",
+      "name": "Salford",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-SLG",
+      "name": "Slough",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-SLK",
+      "name": "South Lanarkshire",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-SND",
+      "name": "Sunderland",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-SOL",
+      "name": "Solihull",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-SOM",
+      "name": "Somerset",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-SOS",
+      "name": "Southend-on-Sea",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-SRY",
+      "name": "Surrey",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-STE",
+      "name": "Stoke-on-Trent",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-STG",
+      "name": "Stirling",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-STH",
+      "name": "Southampton",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-STN",
+      "name": "Sutton",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-STS",
+      "name": "Staffordshire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-STT",
+      "name": "Stockton-on-Tees",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-STY",
+      "name": "South Tyneside",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-SWA",
+      "name": "Swansea [Abertawe GB-ATA]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-SWD",
+      "name": "Swindon",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-SWK",
+      "name": "Southwark",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-TAM",
+      "name": "Tameside",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-TFW",
+      "name": "Telford and Wrekin",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-THR",
+      "name": "Thurrock",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-TOB",
+      "name": "Torbay",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-TOF",
+      "name": "Torfaen [Tor-faen]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-TRF",
+      "name": "Trafford",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-TWH",
+      "name": "Tower Hamlets",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-VGL",
+      "name": "Vale of Glamorgan, The [Bro Morgannwg GB-BMG]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-WAR",
+      "name": "Warwickshire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-WBK",
+      "name": "West Berkshire",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-WDU",
+      "name": "West Dunbartonshire",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-WFT",
+      "name": "Waltham Forest",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-WGN",
+      "name": "Wigan",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-WIL",
+      "name": "Wiltshire",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-WKF",
+      "name": "Wakefield",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-WLL",
+      "name": "Walsall",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-WLN",
+      "name": "West Lothian",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GB-WLS",
+      "name": "Wales [Cymru GB-CYM]",
+      "type": "Country"
+    },
+    {
+      "code": "GB-WLV",
+      "name": "Wolverhampton",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-WND",
+      "name": "Wandsworth",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-WNH",
+      "name": "West Northamptonshire",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-WNM",
+      "name": "Windsor and Maidenhead",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-WOK",
+      "name": "Wokingham",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-WOR",
+      "name": "Worcestershire",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-WRL",
+      "name": "Wirral",
+      "parent": "GB-ENG",
+      "type": "Metropolitan district"
+    },
+    {
+      "code": "GB-WRT",
+      "name": "Warrington",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-WRX",
+      "name": "Wrexham [Wrecsam GB-WRC]",
+      "parent": "GB-WLS",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-WSM",
+      "name": "Westminster",
+      "parent": "GB-ENG",
+      "type": "London borough"
+    },
+    {
+      "code": "GB-WSX",
+      "name": "West Sussex",
+      "parent": "GB-ENG",
+      "type": "Two-tier county"
+    },
+    {
+      "code": "GB-YOR",
+      "name": "York",
+      "parent": "GB-ENG",
+      "type": "Unitary authority"
+    },
+    {
+      "code": "GB-ZET",
+      "name": "Shetland Islands",
+      "parent": "GB-SCT",
+      "type": "Council area"
+    },
+    {
+      "code": "GD-01",
+      "name": "Saint Andrew",
+      "type": "Parish"
+    },
+    {
+      "code": "GD-02",
+      "name": "Saint David",
+      "type": "Parish"
+    },
+    {
+      "code": "GD-03",
+      "name": "Saint George",
+      "type": "Parish"
+    },
+    {
+      "code": "GD-04",
+      "name": "Saint John",
+      "type": "Parish"
+    },
+    {
+      "code": "GD-05",
+      "name": "Saint Mark",
+      "type": "Parish"
+    },
+    {
+      "code": "GD-06",
+      "name": "Saint Patrick",
+      "type": "Parish"
+    },
+    {
+      "code": "GD-10",
+      "name": "Southern Grenadine Islands",
+      "type": "Dependency"
+    },
+    {
+      "code": "GE-AB",
+      "name": "Abkhazia",
+      "type": "Autonomous republic"
+    },
+    {
+      "code": "GE-AJ",
+      "name": "Ajaria",
+      "type": "Autonomous republic"
+    },
+    {
+      "code": "GE-GU",
+      "name": "Guria",
+      "type": "Region"
+    },
+    {
+      "code": "GE-IM",
+      "name": "Imereti",
+      "type": "Region"
+    },
+    {
+      "code": "GE-KA",
+      "name": "K'akheti",
+      "type": "Region"
+    },
+    {
+      "code": "GE-KK",
+      "name": "Kvemo Kartli",
+      "type": "Region"
+    },
+    {
+      "code": "GE-MM",
+      "name": "Mtskheta-Mtianeti",
+      "type": "Region"
+    },
+    {
+      "code": "GE-RL",
+      "name": "Rach'a-Lechkhumi-Kvemo Svaneti",
+      "type": "Region"
+    },
+    {
+      "code": "GE-SJ",
+      "name": "Samtskhe-Javakheti",
+      "type": "Region"
+    },
+    {
+      "code": "GE-SK",
+      "name": "Shida Kartli",
+      "type": "Region"
+    },
+    {
+      "code": "GE-SZ",
+      "name": "Samegrelo-Zemo Svaneti",
+      "type": "Region"
+    },
+    {
+      "code": "GE-TB",
+      "name": "Tbilisi",
+      "type": "City"
+    },
+    {
+      "code": "GH-AA",
+      "name": "Greater Accra",
+      "type": "Region"
+    },
+    {
+      "code": "GH-AF",
+      "name": "Ahafo",
+      "type": "Region"
+    },
+    {
+      "code": "GH-AH",
+      "name": "Ashanti",
+      "type": "Region"
+    },
+    {
+      "code": "GH-BE",
+      "name": "Bono East",
+      "type": "Region"
+    },
+    {
+      "code": "GH-BO",
+      "name": "Bono",
+      "type": "Region"
+    },
+    {
+      "code": "GH-CP",
+      "name": "Central",
+      "type": "Region"
+    },
+    {
+      "code": "GH-EP",
+      "name": "Eastern",
+      "type": "Region"
+    },
+    {
+      "code": "GH-NE",
+      "name": "North East",
+      "type": "Region"
+    },
+    {
+      "code": "GH-NP",
+      "name": "Northern",
+      "type": "Region"
+    },
+    {
+      "code": "GH-OT",
+      "name": "Oti",
+      "type": "Region"
+    },
+    {
+      "code": "GH-SV",
+      "name": "Savannah",
+      "type": "Region"
+    },
+    {
+      "code": "GH-TV",
+      "name": "Volta",
+      "type": "Region"
+    },
+    {
+      "code": "GH-UE",
+      "name": "Upper East",
+      "type": "Region"
+    },
+    {
+      "code": "GH-UW",
+      "name": "Upper West",
+      "type": "Region"
+    },
+    {
+      "code": "GH-WN",
+      "name": "Western North",
+      "type": "Region"
+    },
+    {
+      "code": "GH-WP",
+      "name": "Western",
+      "type": "Region"
+    },
+    {
+      "code": "GL-AV",
+      "name": "Avannaata Kommunia",
+      "type": "Municipality"
+    },
+    {
+      "code": "GL-KU",
+      "name": "Kommune Kujalleq",
+      "type": "Municipality"
+    },
+    {
+      "code": "GL-QE",
+      "name": "Qeqqata Kommunia",
+      "type": "Municipality"
+    },
+    {
+      "code": "GL-QT",
+      "name": "Kommune Qeqertalik",
+      "type": "Municipality"
+    },
+    {
+      "code": "GL-SM",
+      "name": "Kommuneqarfik Sermersooq",
+      "type": "Municipality"
+    },
+    {
+      "code": "GM-B",
+      "name": "Banjul",
+      "type": "City"
+    },
+    {
+      "code": "GM-L",
+      "name": "Lower River",
+      "type": "Division"
+    },
+    {
+      "code": "GM-M",
+      "name": "Central River",
+      "type": "Division"
+    },
+    {
+      "code": "GM-N",
+      "name": "North Bank",
+      "type": "Division"
+    },
+    {
+      "code": "GM-U",
+      "name": "Upper River",
+      "type": "Division"
+    },
+    {
+      "code": "GM-W",
+      "name": "Western",
+      "type": "Division"
+    },
+    {
+      "code": "GN-B",
+      "name": "Boké",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GN-BE",
+      "name": "Beyla",
+      "parent": "GN-N",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-BF",
+      "name": "Boffa",
+      "parent": "GN-B",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-BK",
+      "name": "Boké",
+      "parent": "GN-B",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-C",
+      "name": "Conakry",
+      "type": "Governorate"
+    },
+    {
+      "code": "GN-CO",
+      "name": "Coyah",
+      "parent": "GN-D",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-D",
+      "name": "Kindia",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GN-DB",
+      "name": "Dabola",
+      "parent": "GN-F",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-DI",
+      "name": "Dinguiraye",
+      "parent": "GN-F",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-DL",
+      "name": "Dalaba",
+      "parent": "GN-M",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-DU",
+      "name": "Dubréka",
+      "parent": "GN-D",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-F",
+      "name": "Faranah",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GN-FA",
+      "name": "Faranah",
+      "parent": "GN-F",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-FO",
+      "name": "Forécariah",
+      "parent": "GN-D",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-FR",
+      "name": "Fria",
+      "parent": "GN-B",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-GA",
+      "name": "Gaoual",
+      "parent": "GN-B",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-GU",
+      "name": "Guékédou",
+      "parent": "GN-N",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-K",
+      "name": "Kankan",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GN-KA",
+      "name": "Kankan",
+      "parent": "GN-K",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-KB",
+      "name": "Koubia",
+      "parent": "GN-L",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-KD",
+      "name": "Kindia",
+      "parent": "GN-D",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-KE",
+      "name": "Kérouané",
+      "parent": "GN-K",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-KN",
+      "name": "Koundara",
+      "parent": "GN-B",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-KO",
+      "name": "Kouroussa",
+      "parent": "GN-K",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-KS",
+      "name": "Kissidougou",
+      "parent": "GN-F",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-L",
+      "name": "Labé",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GN-LA",
+      "name": "Labé",
+      "parent": "GN-L",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-LE",
+      "name": "Lélouma",
+      "parent": "GN-L",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-LO",
+      "name": "Lola",
+      "parent": "GN-N",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-M",
+      "name": "Mamou",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GN-MC",
+      "name": "Macenta",
+      "parent": "GN-N",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-MD",
+      "name": "Mandiana",
+      "parent": "GN-K",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-ML",
+      "name": "Mali",
+      "parent": "GN-L",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-MM",
+      "name": "Mamou",
+      "parent": "GN-M",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-N",
+      "name": "Nzérékoré",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GN-NZ",
+      "name": "Nzérékoré",
+      "parent": "GN-N",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-PI",
+      "name": "Pita",
+      "parent": "GN-M",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-SI",
+      "name": "Siguiri",
+      "parent": "GN-K",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-TE",
+      "name": "Télimélé",
+      "parent": "GN-D",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-TO",
+      "name": "Tougué",
+      "parent": "GN-L",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GN-YO",
+      "name": "Yomou",
+      "parent": "GN-N",
+      "type": "Prefecture"
+    },
+    {
+      "code": "GQ-AN",
+      "name": "Annobon",
+      "parent": "GQ-I",
+      "type": "Province"
+    },
+    {
+      "code": "GQ-BN",
+      "name": "Bioko Nord",
+      "parent": "GQ-I",
+      "type": "Province"
+    },
+    {
+      "code": "GQ-BS",
+      "name": "Bioko Sud",
+      "parent": "GQ-I",
+      "type": "Province"
+    },
+    {
+      "code": "GQ-C",
+      "name": "Région Continentale",
+      "type": "Region"
+    },
+    {
+      "code": "GQ-CS",
+      "name": "Centro Sud",
+      "parent": "GQ-C",
+      "type": "Province"
+    },
+    {
+      "code": "GQ-DJ",
+      "name": "Djibloho",
+      "parent": "GQ-C",
+      "type": "Province"
+    },
+    {
+      "code": "GQ-I",
+      "name": "Région Insulaire",
+      "type": "Region"
+    },
+    {
+      "code": "GQ-KN",
+      "name": "Kié-Ntem",
+      "parent": "GQ-C",
+      "type": "Province"
+    },
+    {
+      "code": "GQ-LI",
+      "name": "Littoral",
+      "parent": "GQ-C",
+      "type": "Province"
+    },
+    {
+      "code": "GQ-WN",
+      "name": "Wele-Nzas",
+      "parent": "GQ-C",
+      "type": "Province"
+    },
+    {
+      "code": "GR-69",
+      "name": "Ágion Óros",
+      "type": "Self-governed part"
+    },
+    {
+      "code": "GR-A",
+      "name": "Anatolikí Makedonía kai Thráki",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GR-B",
+      "name": "Kentrikí Makedonía",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GR-C",
+      "name": "Dytikí Makedonía",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GR-D",
+      "name": "Ípeiros",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GR-E",
+      "name": "Thessalía",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GR-F",
+      "name": "Ionía Nísia",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GR-G",
+      "name": "Dytikí Elláda",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GR-H",
+      "name": "Stereá Elláda",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GR-I",
+      "name": "Attikí",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GR-J",
+      "name": "Pelopónnisos",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GR-K",
+      "name": "Vóreio Aigaío",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GR-L",
+      "name": "Nótio Aigaío",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GR-M",
+      "name": "Kríti",
+      "type": "Administrative region"
+    },
+    {
+      "code": "GT-01",
+      "name": "Guatemala",
+      "type": "Department"
+    },
+    {
+      "code": "GT-02",
+      "name": "El Progreso",
+      "type": "Department"
+    },
+    {
+      "code": "GT-03",
+      "name": "Sacatepéquez",
+      "type": "Department"
+    },
+    {
+      "code": "GT-04",
+      "name": "Chimaltenango",
+      "type": "Department"
+    },
+    {
+      "code": "GT-05",
+      "name": "Escuintla",
+      "type": "Department"
+    },
+    {
+      "code": "GT-06",
+      "name": "Santa Rosa",
+      "type": "Department"
+    },
+    {
+      "code": "GT-07",
+      "name": "Sololá",
+      "type": "Department"
+    },
+    {
+      "code": "GT-08",
+      "name": "Totonicapán",
+      "type": "Department"
+    },
+    {
+      "code": "GT-09",
+      "name": "Quetzaltenango",
+      "type": "Department"
+    },
+    {
+      "code": "GT-10",
+      "name": "Suchitepéquez",
+      "type": "Department"
+    },
+    {
+      "code": "GT-11",
+      "name": "Retalhuleu",
+      "type": "Department"
+    },
+    {
+      "code": "GT-12",
+      "name": "San Marcos",
+      "type": "Department"
+    },
+    {
+      "code": "GT-13",
+      "name": "Huehuetenango",
+      "type": "Department"
+    },
+    {
+      "code": "GT-14",
+      "name": "Quiché",
+      "type": "Department"
+    },
+    {
+      "code": "GT-15",
+      "name": "Baja Verapaz",
+      "type": "Department"
+    },
+    {
+      "code": "GT-16",
+      "name": "Alta Verapaz",
+      "type": "Department"
+    },
+    {
+      "code": "GT-17",
+      "name": "Petén",
+      "type": "Department"
+    },
+    {
+      "code": "GT-18",
+      "name": "Izabal",
+      "type": "Department"
+    },
+    {
+      "code": "GT-19",
+      "name": "Zacapa",
+      "type": "Department"
+    },
+    {
+      "code": "GT-20",
+      "name": "Chiquimula",
+      "type": "Department"
+    },
+    {
+      "code": "GT-21",
+      "name": "Jalapa",
+      "type": "Department"
+    },
+    {
+      "code": "GT-22",
+      "name": "Jutiapa",
+      "type": "Department"
+    },
+    {
+      "code": "GW-BA",
+      "name": "Bafatá",
+      "parent": "GW-L",
+      "type": "Region"
+    },
+    {
+      "code": "GW-BL",
+      "name": "Bolama / Bijagós",
+      "parent": "GW-S",
+      "type": "Region"
+    },
+    {
+      "code": "GW-BM",
+      "name": "Biombo",
+      "parent": "GW-N",
+      "type": "Region"
+    },
+    {
+      "code": "GW-BS",
+      "name": "Bissau",
+      "type": "Autonomous sector"
+    },
+    {
+      "code": "GW-CA",
+      "name": "Cacheu",
+      "parent": "GW-N",
+      "type": "Region"
+    },
+    {
+      "code": "GW-GA",
+      "name": "Gabú",
+      "parent": "GW-L",
+      "type": "Region"
+    },
+    {
+      "code": "GW-L",
+      "name": "Leste",
+      "type": "Province"
+    },
+    {
+      "code": "GW-N",
+      "name": "Norte",
+      "type": "Province"
+    },
+    {
+      "code": "GW-OI",
+      "name": "Oio",
+      "parent": "GW-N",
+      "type": "Region"
+    },
+    {
+      "code": "GW-QU",
+      "name": "Quinara",
+      "parent": "GW-S",
+      "type": "Region"
+    },
+    {
+      "code": "GW-S",
+      "name": "Sul",
+      "type": "Province"
+    },
+    {
+      "code": "GW-TO",
+      "name": "Tombali",
+      "parent": "GW-S",
+      "type": "Region"
+    },
+    {
+      "code": "GY-BA",
+      "name": "Barima-Waini",
+      "type": "Region"
+    },
+    {
+      "code": "GY-CU",
+      "name": "Cuyuni-Mazaruni",
+      "type": "Region"
+    },
+    {
+      "code": "GY-DE",
+      "name": "Demerara-Mahaica",
+      "type": "Region"
+    },
+    {
+      "code": "GY-EB",
+      "name": "East Berbice-Corentyne",
+      "type": "Region"
+    },
+    {
+      "code": "GY-ES",
+      "name": "Essequibo Islands-West Demerara",
+      "type": "Region"
+    },
+    {
+      "code": "GY-MA",
+      "name": "Mahaica-Berbice",
+      "type": "Region"
+    },
+    {
+      "code": "GY-PM",
+      "name": "Pomeroon-Supenaam",
+      "type": "Region"
+    },
+    {
+      "code": "GY-PT",
+      "name": "Potaro-Siparuni",
+      "type": "Region"
+    },
+    {
+      "code": "GY-UD",
+      "name": "Upper Demerara-Berbice",
+      "type": "Region"
+    },
+    {
+      "code": "GY-UT",
+      "name": "Upper Takutu-Upper Essequibo",
+      "type": "Region"
+    },
+    {
+      "code": "HN-AT",
+      "name": "Atlántida",
+      "type": "Department"
+    },
+    {
+      "code": "HN-CH",
+      "name": "Choluteca",
+      "type": "Department"
+    },
+    {
+      "code": "HN-CL",
+      "name": "Colón",
+      "type": "Department"
+    },
+    {
+      "code": "HN-CM",
+      "name": "Comayagua",
+      "type": "Department"
+    },
+    {
+      "code": "HN-CP",
+      "name": "Copán",
+      "type": "Department"
+    },
+    {
+      "code": "HN-CR",
+      "name": "Cortés",
+      "type": "Department"
+    },
+    {
+      "code": "HN-EP",
+      "name": "El Paraíso",
+      "type": "Department"
+    },
+    {
+      "code": "HN-FM",
+      "name": "Francisco Morazán",
+      "type": "Department"
+    },
+    {
+      "code": "HN-GD",
+      "name": "Gracias a Dios",
+      "type": "Department"
+    },
+    {
+      "code": "HN-IB",
+      "name": "Islas de la Bahía",
+      "type": "Department"
+    },
+    {
+      "code": "HN-IN",
+      "name": "Intibucá",
+      "type": "Department"
+    },
+    {
+      "code": "HN-LE",
+      "name": "Lempira",
+      "type": "Department"
+    },
+    {
+      "code": "HN-LP",
+      "name": "La Paz",
+      "type": "Department"
+    },
+    {
+      "code": "HN-OC",
+      "name": "Ocotepeque",
+      "type": "Department"
+    },
+    {
+      "code": "HN-OL",
+      "name": "Olancho",
+      "type": "Department"
+    },
+    {
+      "code": "HN-SB",
+      "name": "Santa Bárbara",
+      "type": "Department"
+    },
+    {
+      "code": "HN-VA",
+      "name": "Valle",
+      "type": "Department"
+    },
+    {
+      "code": "HN-YO",
+      "name": "Yoro",
+      "type": "Department"
+    },
+    {
+      "code": "HR-01",
+      "name": "Zagrebačka županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-02",
+      "name": "Krapinsko-zagorska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-03",
+      "name": "Sisačko-moslavačka županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-04",
+      "name": "Karlovačka županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-05",
+      "name": "Varaždinska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-06",
+      "name": "Koprivničko-križevačka županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-07",
+      "name": "Bjelovarsko-bilogorska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-08",
+      "name": "Primorsko-goranska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-09",
+      "name": "Ličko-senjska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-10",
+      "name": "Virovitičko-podravska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-11",
+      "name": "Požeško-slavonska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-12",
+      "name": "Brodsko-posavska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-13",
+      "name": "Zadarska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-14",
+      "name": "Osječko-baranjska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-15",
+      "name": "Šibensko-kninska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-16",
+      "name": "Vukovarsko-srijemska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-17",
+      "name": "Splitsko-dalmatinska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-18",
+      "name": "Istarska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-19",
+      "name": "Dubrovačko-neretvanska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-20",
+      "name": "Međimurska županija",
+      "type": "County"
+    },
+    {
+      "code": "HR-21",
+      "name": "Grad Zagreb",
+      "type": "City"
+    },
+    {
+      "code": "HT-AR",
+      "name": "Artibonite",
+      "type": "Department"
+    },
+    {
+      "code": "HT-CE",
+      "name": "Centre",
+      "type": "Department"
+    },
+    {
+      "code": "HT-GA",
+      "name": "Grande’Anse",
+      "type": "Department"
+    },
+    {
+      "code": "HT-ND",
+      "name": "Nord",
+      "type": "Department"
+    },
+    {
+      "code": "HT-NE",
+      "name": "Nord-Est",
+      "type": "Department"
+    },
+    {
+      "code": "HT-NI",
+      "name": "Nippes",
+      "type": "Department"
+    },
+    {
+      "code": "HT-NO",
+      "name": "Nord-Ouest",
+      "type": "Department"
+    },
+    {
+      "code": "HT-OU",
+      "name": "Ouest",
+      "type": "Department"
+    },
+    {
+      "code": "HT-SD",
+      "name": "Sud",
+      "type": "Department"
+    },
+    {
+      "code": "HT-SE",
+      "name": "Sud-Est",
+      "type": "Department"
+    },
+    {
+      "code": "HU-BA",
+      "name": "Baranya",
+      "type": "County"
+    },
+    {
+      "code": "HU-BC",
+      "name": "Békéscsaba",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-BE",
+      "name": "Békés",
+      "type": "County"
+    },
+    {
+      "code": "HU-BK",
+      "name": "Bács-Kiskun",
+      "type": "County"
+    },
+    {
+      "code": "HU-BU",
+      "name": "Budapest",
+      "type": "Capital city"
+    },
+    {
+      "code": "HU-BZ",
+      "name": "Borsod-Abaúj-Zemplén",
+      "type": "County"
+    },
+    {
+      "code": "HU-CS",
+      "name": "Csongrád-Csanád",
+      "type": "County"
+    },
+    {
+      "code": "HU-DE",
+      "name": "Debrecen",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-DU",
+      "name": "Dunaújváros",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-EG",
+      "name": "Eger",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-ER",
+      "name": "Érd",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-FE",
+      "name": "Fejér",
+      "type": "County"
+    },
+    {
+      "code": "HU-GS",
+      "name": "Győr-Moson-Sopron",
+      "type": "County"
+    },
+    {
+      "code": "HU-GY",
+      "name": "Győr",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-HB",
+      "name": "Hajdú-Bihar",
+      "type": "County"
+    },
+    {
+      "code": "HU-HE",
+      "name": "Heves",
+      "type": "County"
+    },
+    {
+      "code": "HU-HV",
+      "name": "Hódmezővásárhely",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-JN",
+      "name": "Jász-Nagykun-Szolnok",
+      "type": "County"
+    },
+    {
+      "code": "HU-KE",
+      "name": "Komárom-Esztergom",
+      "type": "County"
+    },
+    {
+      "code": "HU-KM",
+      "name": "Kecskemét",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-KV",
+      "name": "Kaposvár",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-MI",
+      "name": "Miskolc",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-NK",
+      "name": "Nagykanizsa",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-NO",
+      "name": "Nógrád",
+      "type": "County"
+    },
+    {
+      "code": "HU-NY",
+      "name": "Nyíregyháza",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-PE",
+      "name": "Pest",
+      "type": "County"
+    },
+    {
+      "code": "HU-PS",
+      "name": "Pécs",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-SD",
+      "name": "Szeged",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-SF",
+      "name": "Székesfehérvár",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-SH",
+      "name": "Szombathely",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-SK",
+      "name": "Szolnok",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-SN",
+      "name": "Sopron",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-SO",
+      "name": "Somogy",
+      "type": "County"
+    },
+    {
+      "code": "HU-SS",
+      "name": "Szekszárd",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-ST",
+      "name": "Salgótarján",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-SZ",
+      "name": "Szabolcs-Szatmár-Bereg",
+      "type": "County"
+    },
+    {
+      "code": "HU-TB",
+      "name": "Tatabánya",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-TO",
+      "name": "Tolna",
+      "type": "County"
+    },
+    {
+      "code": "HU-VA",
+      "name": "Vas",
+      "type": "County"
+    },
+    {
+      "code": "HU-VE",
+      "name": "Veszprém",
+      "type": "County"
+    },
+    {
+      "code": "HU-VM",
+      "name": "Veszprém",
+      "type": "City with county rights"
+    },
+    {
+      "code": "HU-ZA",
+      "name": "Zala",
+      "type": "County"
+    },
+    {
+      "code": "HU-ZE",
+      "name": "Zalaegerszeg",
+      "type": "City with county rights"
+    },
+    {
+      "code": "ID-AC",
+      "name": "Aceh",
+      "parent": "ID-SM",
+      "type": "Province"
+    },
+    {
+      "code": "ID-BA",
+      "name": "Bali",
+      "parent": "ID-NU",
+      "type": "Province"
+    },
+    {
+      "code": "ID-BB",
+      "name": "Kepulauan Bangka Belitung",
+      "parent": "ID-SM",
+      "type": "Province"
+    },
+    {
+      "code": "ID-BE",
+      "name": "Bengkulu",
+      "parent": "ID-SM",
+      "type": "Province"
+    },
+    {
+      "code": "ID-BT",
+      "name": "Banten",
+      "parent": "ID-JW",
+      "type": "Province"
+    },
+    {
+      "code": "ID-GO",
+      "name": "Gorontalo",
+      "parent": "ID-SL",
+      "type": "Province"
+    },
+    {
+      "code": "ID-JA",
+      "name": "Jambi",
+      "parent": "ID-SM",
+      "type": "Province"
+    },
+    {
+      "code": "ID-JB",
+      "name": "Jawa Barat",
+      "parent": "ID-JW",
+      "type": "Province"
+    },
+    {
+      "code": "ID-JI",
+      "name": "Jawa Timur",
+      "parent": "ID-JW",
+      "type": "Province"
+    },
+    {
+      "code": "ID-JK",
+      "name": "Jakarta Raya",
+      "parent": "ID-JW",
+      "type": "Capital district"
+    },
+    {
+      "code": "ID-JT",
+      "name": "Jawa Tengah",
+      "parent": "ID-JW",
+      "type": "Province"
+    },
+    {
+      "code": "ID-JW",
+      "name": "Jawa",
+      "type": "Geographical unit"
+    },
+    {
+      "code": "ID-KA",
+      "name": "Kalimantan",
+      "type": "Geographical unit"
+    },
+    {
+      "code": "ID-KB",
+      "name": "Kalimantan Barat",
+      "parent": "ID-KA",
+      "type": "Province"
+    },
+    {
+      "code": "ID-KI",
+      "name": "Kalimantan Timur",
+      "parent": "ID-KA",
+      "type": "Province"
+    },
+    {
+      "code": "ID-KR",
+      "name": "Kepulauan Riau",
+      "parent": "ID-SM",
+      "type": "Province"
+    },
+    {
+      "code": "ID-KS",
+      "name": "Kalimantan Selatan",
+      "parent": "ID-KA",
+      "type": "Province"
+    },
+    {
+      "code": "ID-KT",
+      "name": "Kalimantan Tengah",
+      "parent": "ID-KA",
+      "type": "Province"
+    },
+    {
+      "code": "ID-KU",
+      "name": "Kalimantan Utara",
+      "parent": "ID-KA",
+      "type": "Province"
+    },
+    {
+      "code": "ID-LA",
+      "name": "Lampung",
+      "parent": "ID-SM",
+      "type": "Province"
+    },
+    {
+      "code": "ID-MA",
+      "name": "Maluku",
+      "parent": "ID-ML",
+      "type": "Province"
+    },
+    {
+      "code": "ID-ML",
+      "name": "Maluku",
+      "type": "Geographical unit"
+    },
+    {
+      "code": "ID-MU",
+      "name": "Maluku Utara",
+      "parent": "ID-ML",
+      "type": "Province"
+    },
+    {
+      "code": "ID-NB",
+      "name": "Nusa Tenggara Barat",
+      "parent": "ID-NU",
+      "type": "Province"
+    },
+    {
+      "code": "ID-NT",
+      "name": "Nusa Tenggara Timur",
+      "parent": "ID-NU",
+      "type": "Province"
+    },
+    {
+      "code": "ID-NU",
+      "name": "Nusa Tenggara",
+      "type": "Geographical unit"
+    },
+    {
+      "code": "ID-PA",
+      "name": "Papua",
+      "parent": "ID-PP",
+      "type": "Province"
+    },
+    {
+      "code": "ID-PB",
+      "name": "Papua Barat",
+      "parent": "ID-PP",
+      "type": "Province"
+    },
+    {
+      "code": "ID-PD",
+      "name": "Papua Barat Daya",
+      "parent": "ID-PP",
+      "type": "Province"
+    },
+    {
+      "code": "ID-PE",
+      "name": "Papua Pengunungan",
+      "parent": "ID-PP",
+      "type": "Province"
+    },
+    {
+      "code": "ID-PP",
+      "name": "Papua",
+      "type": "Geographical unit"
+    },
+    {
+      "code": "ID-PS",
+      "name": "Papua Selatan",
+      "parent": "ID-PP",
+      "type": "Province"
+    },
+    {
+      "code": "ID-PT",
+      "name": "Papua Tengah",
+      "parent": "ID-PP",
+      "type": "Province"
+    },
+    {
+      "code": "ID-RI",
+      "name": "Riau",
+      "parent": "ID-SM",
+      "type": "Province"
+    },
+    {
+      "code": "ID-SA",
+      "name": "Sulawesi Utara",
+      "parent": "ID-SL",
+      "type": "Province"
+    },
+    {
+      "code": "ID-SB",
+      "name": "Sumatera Barat",
+      "parent": "ID-SM",
+      "type": "Province"
+    },
+    {
+      "code": "ID-SG",
+      "name": "Sulawesi Tenggara",
+      "parent": "ID-SL",
+      "type": "Province"
+    },
+    {
+      "code": "ID-SL",
+      "name": "Sulawesi",
+      "type": "Geographical unit"
+    },
+    {
+      "code": "ID-SM",
+      "name": "Sumatera",
+      "type": "Geographical unit"
+    },
+    {
+      "code": "ID-SN",
+      "name": "Sulawesi Selatan",
+      "parent": "ID-SL",
+      "type": "Province"
+    },
+    {
+      "code": "ID-SR",
+      "name": "Sulawesi Barat",
+      "parent": "ID-SL",
+      "type": "Province"
+    },
+    {
+      "code": "ID-SS",
+      "name": "Sumatera Selatan",
+      "parent": "ID-SM",
+      "type": "Province"
+    },
+    {
+      "code": "ID-ST",
+      "name": "Sulawesi Tengah",
+      "parent": "ID-SL",
+      "type": "Province"
+    },
+    {
+      "code": "ID-SU",
+      "name": "Sumatera Utara",
+      "parent": "ID-SM",
+      "type": "Province"
+    },
+    {
+      "code": "ID-YO",
+      "name": "Yogyakarta",
+      "parent": "ID-JW",
+      "type": "Special region"
+    },
+    {
+      "code": "IE-C",
+      "name": "Connaught",
+      "type": "Province"
+    },
+    {
+      "code": "IE-CE",
+      "name": "Clare",
+      "parent": "IE-M",
+      "type": "County"
+    },
+    {
+      "code": "IE-CN",
+      "name": "Cavan",
+      "parent": "IE-U",
+      "type": "County"
+    },
+    {
+      "code": "IE-CO",
+      "name": "Cork",
+      "parent": "IE-M",
+      "type": "County"
+    },
+    {
+      "code": "IE-CW",
+      "name": "Carlow",
+      "parent": "IE-L",
+      "type": "County"
+    },
+    {
+      "code": "IE-D",
+      "name": "Dublin",
+      "parent": "IE-L",
+      "type": "County"
+    },
+    {
+      "code": "IE-DL",
+      "name": "Donegal",
+      "parent": "IE-U",
+      "type": "County"
+    },
+    {
+      "code": "IE-G",
+      "name": "Galway",
+      "parent": "IE-C",
+      "type": "County"
+    },
+    {
+      "code": "IE-KE",
+      "name": "Kildare",
+      "parent": "IE-L",
+      "type": "County"
+    },
+    {
+      "code": "IE-KK",
+      "name": "Kilkenny",
+      "parent": "IE-L",
+      "type": "County"
+    },
+    {
+      "code": "IE-KY",
+      "name": "Kerry",
+      "parent": "IE-M",
+      "type": "County"
+    },
+    {
+      "code": "IE-L",
+      "name": "Leinster",
+      "type": "Province"
+    },
+    {
+      "code": "IE-LD",
+      "name": "Longford",
+      "parent": "IE-L",
+      "type": "County"
+    },
+    {
+      "code": "IE-LH",
+      "name": "Louth",
+      "parent": "IE-L",
+      "type": "County"
+    },
+    {
+      "code": "IE-LK",
+      "name": "Limerick",
+      "parent": "IE-M",
+      "type": "County"
+    },
+    {
+      "code": "IE-LM",
+      "name": "Leitrim",
+      "parent": "IE-C",
+      "type": "County"
+    },
+    {
+      "code": "IE-LS",
+      "name": "Laois",
+      "parent": "IE-L",
+      "type": "County"
+    },
+    {
+      "code": "IE-M",
+      "name": "Munster",
+      "type": "Province"
+    },
+    {
+      "code": "IE-MH",
+      "name": "Meath",
+      "parent": "IE-L",
+      "type": "County"
+    },
+    {
+      "code": "IE-MN",
+      "name": "Monaghan",
+      "parent": "IE-U",
+      "type": "County"
+    },
+    {
+      "code": "IE-MO",
+      "name": "Mayo",
+      "parent": "IE-C",
+      "type": "County"
+    },
+    {
+      "code": "IE-OY",
+      "name": "Offaly",
+      "parent": "IE-L",
+      "type": "County"
+    },
+    {
+      "code": "IE-RN",
+      "name": "Roscommon",
+      "parent": "IE-C",
+      "type": "County"
+    },
+    {
+      "code": "IE-SO",
+      "name": "Sligo",
+      "parent": "IE-C",
+      "type": "County"
+    },
+    {
+      "code": "IE-TA",
+      "name": "Tipperary",
+      "parent": "IE-M",
+      "type": "County"
+    },
+    {
+      "code": "IE-U",
+      "name": "Ulster",
+      "type": "Province"
+    },
+    {
+      "code": "IE-WD",
+      "name": "Waterford",
+      "parent": "IE-M",
+      "type": "County"
+    },
+    {
+      "code": "IE-WH",
+      "name": "Westmeath",
+      "parent": "IE-L",
+      "type": "County"
+    },
+    {
+      "code": "IE-WW",
+      "name": "Wicklow",
+      "parent": "IE-L",
+      "type": "County"
+    },
+    {
+      "code": "IE-WX",
+      "name": "Wexford",
+      "parent": "IE-L",
+      "type": "County"
+    },
+    {
+      "code": "IL-D",
+      "name": "Al Janūbī",
+      "type": "District"
+    },
+    {
+      "code": "IL-HA",
+      "name": "H̱efa",
+      "type": "District"
+    },
+    {
+      "code": "IL-JM",
+      "name": "Al Quds",
+      "type": "District"
+    },
+    {
+      "code": "IL-M",
+      "name": "Al Awsaţ",
+      "type": "District"
+    },
+    {
+      "code": "IL-TA",
+      "name": "Tall Abīb",
+      "type": "District"
+    },
+    {
+      "code": "IL-Z",
+      "name": "Ash Shamālī",
+      "type": "District"
+    },
+    {
+      "code": "IN-AN",
+      "name": "Andaman and Nicobar Islands",
+      "type": "Union territory"
+    },
+    {
+      "code": "IN-AP",
+      "name": "Andhra Pradesh",
+      "type": "State"
+    },
+    {
+      "code": "IN-AR",
+      "name": "Arunāchal Pradesh",
+      "type": "State"
+    },
+    {
+      "code": "IN-AS",
+      "name": "Assam",
+      "type": "State"
+    },
+    {
+      "code": "IN-BR",
+      "name": "Bihār",
+      "type": "State"
+    },
+    {
+      "code": "IN-CG",
+      "name": "Chhattīsgarh",
+      "type": "State"
+    },
+    {
+      "code": "IN-CH",
+      "name": "Chandīgarh",
+      "type": "Union territory"
+    },
+    {
+      "code": "IN-DH",
+      "name": "Dādra and Nagar Haveli and Damān and Diu",
+      "type": "Union territory"
+    },
+    {
+      "code": "IN-DL",
+      "name": "Delhi",
+      "type": "Union territory"
+    },
+    {
+      "code": "IN-GA",
+      "name": "Goa",
+      "type": "State"
+    },
+    {
+      "code": "IN-GJ",
+      "name": "Gujarāt",
+      "type": "State"
+    },
+    {
+      "code": "IN-HP",
+      "name": "Himāchal Pradesh",
+      "type": "State"
+    },
+    {
+      "code": "IN-HR",
+      "name": "Haryāna",
+      "type": "State"
+    },
+    {
+      "code": "IN-JH",
+      "name": "Jhārkhand",
+      "type": "State"
+    },
+    {
+      "code": "IN-JK",
+      "name": "Jammu and Kashmīr",
+      "type": "Union territory"
+    },
+    {
+      "code": "IN-KA",
+      "name": "Karnātaka",
+      "type": "State"
+    },
+    {
+      "code": "IN-KL",
+      "name": "Kerala",
+      "type": "State"
+    },
+    {
+      "code": "IN-LA",
+      "name": "Ladākh",
+      "type": "Union territory"
+    },
+    {
+      "code": "IN-LD",
+      "name": "Lakshadweep",
+      "type": "Union territory"
+    },
+    {
+      "code": "IN-MH",
+      "name": "Mahārāshtra",
+      "type": "State"
+    },
+    {
+      "code": "IN-ML",
+      "name": "Meghālaya",
+      "type": "State"
+    },
+    {
+      "code": "IN-MN",
+      "name": "Manipur",
+      "type": "State"
+    },
+    {
+      "code": "IN-MP",
+      "name": "Madhya Pradesh",
+      "type": "State"
+    },
+    {
+      "code": "IN-MZ",
+      "name": "Mizoram",
+      "type": "State"
+    },
+    {
+      "code": "IN-NL",
+      "name": "Nāgāland",
+      "type": "State"
+    },
+    {
+      "code": "IN-OD",
+      "name": "Odisha",
+      "type": "State"
+    },
+    {
+      "code": "IN-PB",
+      "name": "Punjab",
+      "type": "State"
+    },
+    {
+      "code": "IN-PY",
+      "name": "Puducherry",
+      "type": "Union territory"
+    },
+    {
+      "code": "IN-RJ",
+      "name": "Rājasthān",
+      "type": "State"
+    },
+    {
+      "code": "IN-SK",
+      "name": "Sikkim",
+      "type": "State"
+    },
+    {
+      "code": "IN-TN",
+      "name": "Tamil Nādu",
+      "type": "State"
+    },
+    {
+      "code": "IN-TR",
+      "name": "Tripura",
+      "type": "State"
+    },
+    {
+      "code": "IN-TS",
+      "name": "Telangāna",
+      "type": "State"
+    },
+    {
+      "code": "IN-UK",
+      "name": "Uttarākhand",
+      "type": "State"
+    },
+    {
+      "code": "IN-UP",
+      "name": "Uttar Pradesh",
+      "type": "State"
+    },
+    {
+      "code": "IN-WB",
+      "name": "West Bengal",
+      "type": "State"
+    },
+    {
+      "code": "IQ-AN",
+      "name": "Al Anbār",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-AR",
+      "name": "Arbīl",
+      "parent": "IQ-KR",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-BA",
+      "name": "Al Başrah",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-BB",
+      "name": "Bābil",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-BG",
+      "name": "Baghdād",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-DA",
+      "name": "Dahūk",
+      "parent": "IQ-KR",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-DI",
+      "name": "Diyālá",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-DQ",
+      "name": "Dhī Qār",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-KA",
+      "name": "Karbalā’",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-KI",
+      "name": "Kirkūk",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-KR",
+      "name": "Herêm-î Kurdistan",
+      "type": "Region"
+    },
+    {
+      "code": "IQ-MA",
+      "name": "Maysān",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-MU",
+      "name": "Al Muthanná",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-NA",
+      "name": "An Najaf",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-NI",
+      "name": "Nīnawá",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-QA",
+      "name": "Al Qādisīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-SD",
+      "name": "Şalāḩ ad Dīn",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-SU",
+      "name": "As Sulaymānīyah",
+      "parent": "IQ-KR",
+      "type": "Governorate"
+    },
+    {
+      "code": "IQ-WA",
+      "name": "Wāsiţ",
+      "type": "Governorate"
+    },
+    {
+      "code": "IR-00",
+      "name": "Markazī",
+      "type": "Province"
+    },
+    {
+      "code": "IR-01",
+      "name": "Gīlān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-02",
+      "name": "Māzandarān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-03",
+      "name": "Āz̄ārbāyjān-e Shārqī",
+      "type": "Province"
+    },
+    {
+      "code": "IR-04",
+      "name": "Āz̄ārbāyjān-e Ghārbī",
+      "type": "Province"
+    },
+    {
+      "code": "IR-05",
+      "name": "Kermānshāh",
+      "type": "Province"
+    },
+    {
+      "code": "IR-06",
+      "name": "Khūzestān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-07",
+      "name": "Fārs",
+      "type": "Province"
+    },
+    {
+      "code": "IR-08",
+      "name": "Kermān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-09",
+      "name": "Khorāsān-e Raẕavī",
+      "type": "Province"
+    },
+    {
+      "code": "IR-10",
+      "name": "Eşfahān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-11",
+      "name": "Sīstān va Balūchestān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-12",
+      "name": "Kordestān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-13",
+      "name": "Hamadān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-14",
+      "name": "Chahār Maḩāl va Bakhtīārī",
+      "type": "Province"
+    },
+    {
+      "code": "IR-15",
+      "name": "Lorestān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-16",
+      "name": "Īlām",
+      "type": "Province"
+    },
+    {
+      "code": "IR-17",
+      "name": "Kohgīlūyeh va Bowyer Aḩmad",
+      "type": "Province"
+    },
+    {
+      "code": "IR-18",
+      "name": "Būshehr",
+      "type": "Province"
+    },
+    {
+      "code": "IR-19",
+      "name": "Zanjān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-20",
+      "name": "Semnān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-21",
+      "name": "Yazd",
+      "type": "Province"
+    },
+    {
+      "code": "IR-22",
+      "name": "Hormozgān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-23",
+      "name": "Tehrān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-24",
+      "name": "Ardabīl",
+      "type": "Province"
+    },
+    {
+      "code": "IR-25",
+      "name": "Qom",
+      "type": "Province"
+    },
+    {
+      "code": "IR-26",
+      "name": "Qazvīn",
+      "type": "Province"
+    },
+    {
+      "code": "IR-27",
+      "name": "Golestān",
+      "type": "Province"
+    },
+    {
+      "code": "IR-28",
+      "name": "Khorāsān-e Shomālī",
+      "type": "Province"
+    },
+    {
+      "code": "IR-29",
+      "name": "Khorāsān-e Jonūbī",
+      "type": "Province"
+    },
+    {
+      "code": "IR-30",
+      "name": "Alborz",
+      "type": "Province"
+    },
+    {
+      "code": "IS-1",
+      "name": "Höfuðborgarsvæði",
+      "type": "Region"
+    },
+    {
+      "code": "IS-2",
+      "name": "Suðurnes",
+      "type": "Region"
+    },
+    {
+      "code": "IS-3",
+      "name": "Vesturland",
+      "type": "Region"
+    },
+    {
+      "code": "IS-4",
+      "name": "Vestfirðir",
+      "type": "Region"
+    },
+    {
+      "code": "IS-5",
+      "name": "Norðurland vestra",
+      "type": "Region"
+    },
+    {
+      "code": "IS-6",
+      "name": "Norðurland eystra",
+      "type": "Region"
+    },
+    {
+      "code": "IS-7",
+      "name": "Austurland",
+      "type": "Region"
+    },
+    {
+      "code": "IS-8",
+      "name": "Suðurland",
+      "type": "Region"
+    },
+    {
+      "code": "IS-AKN",
+      "name": "Akraneskaupstaður",
+      "parent": "IS-3",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-AKU",
+      "name": "Akureyrarbær",
+      "parent": "IS-6",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-ARN",
+      "name": "Árneshreppur",
+      "parent": "IS-4",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-ASA",
+      "name": "Ásahreppur",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-BLA",
+      "name": "Bláskógabyggð",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-BOG",
+      "name": "Borgarbyggð",
+      "parent": "IS-3",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-BOL",
+      "name": "Bolungarvíkurkaupstaður",
+      "parent": "IS-4",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-DAB",
+      "name": "Dalabyggð",
+      "parent": "IS-3",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-DAV",
+      "name": "Dalvíkurbyggð",
+      "parent": "IS-6",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-EOM",
+      "name": "Eyja- og Miklaholtshreppur",
+      "parent": "IS-3",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-EYF",
+      "name": "Eyjafjarðarsveit",
+      "parent": "IS-6",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-FJD",
+      "name": "Fjarðabyggð",
+      "parent": "IS-7",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-FJL",
+      "name": "Fjallabyggð",
+      "parent": "IS-6",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-FLA",
+      "name": "Flóahreppur",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-FLR",
+      "name": "Fljótsdalshreppur",
+      "parent": "IS-7",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-GAR",
+      "name": "Garðabær",
+      "parent": "IS-1",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-GOG",
+      "name": "Grímsnes- og Grafningshreppur",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-GRN",
+      "name": "Grindavíkurbær",
+      "parent": "IS-2",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-GRU",
+      "name": "Grundarfjarðarbær",
+      "parent": "IS-3",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-GRY",
+      "name": "Grýtubakkahreppur",
+      "parent": "IS-6",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-HAF",
+      "name": "Hafnarfjarðarkaupstaður",
+      "parent": "IS-1",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-HRG",
+      "name": "Hörgársveit",
+      "parent": "IS-6",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-HRU",
+      "name": "Hrunamannahreppur",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-HUG",
+      "name": "Húnabyggð",
+      "parent": "IS-5",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-HUV",
+      "name": "Húnaþing vestra",
+      "parent": "IS-5",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-HVA",
+      "name": "Hvalfjarðarsveit",
+      "parent": "IS-3",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-HVE",
+      "name": "Hveragerðisbær",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-ISA",
+      "name": "Ísafjarðarbær",
+      "parent": "IS-4",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-KAL",
+      "name": "Kaldrananeshreppur",
+      "parent": "IS-4",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-KJO",
+      "name": "Kjósarhreppur",
+      "parent": "IS-1",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-KOP",
+      "name": "Kópavogsbær",
+      "parent": "IS-1",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-LAN",
+      "name": "Langanesbyggð",
+      "parent": "IS-6",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-MOS",
+      "name": "Mosfellsbær",
+      "parent": "IS-1",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-MUL",
+      "name": "Múlaþing",
+      "parent": "IS-7",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-MYR",
+      "name": "Mýrdalshreppur",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-NOR",
+      "name": "Norðurþing",
+      "parent": "IS-6",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-RGE",
+      "name": "Rangárþing eystra",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-RGY",
+      "name": "Rangárþing ytra",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-RHH",
+      "name": "Reykhólahreppur",
+      "parent": "IS-4",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-RKN",
+      "name": "Reykjanesbær",
+      "parent": "IS-2",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-RKV",
+      "name": "Reykjavíkurborg",
+      "parent": "IS-1",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SBT",
+      "name": "Svalbarðsstrandarhreppur",
+      "parent": "IS-6",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SDN",
+      "name": "Suðurnesjabær",
+      "parent": "IS-2",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SDV",
+      "name": "Súðavíkurhreppur",
+      "parent": "IS-4",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SEL",
+      "name": "Seltjarnarnesbær",
+      "parent": "IS-1",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SFA",
+      "name": "Sveitarfélagið Árborg",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SHF",
+      "name": "Sveitarfélagið Hornafjörður",
+      "parent": "IS-7",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SKF",
+      "name": "Skaftárhreppur",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SKG",
+      "name": "Skagabyggð",
+      "parent": "IS-5",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SKO",
+      "name": "Skorradalshreppur",
+      "parent": "IS-3",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SKR",
+      "name": "Skagafjörður",
+      "parent": "IS-5",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SNF",
+      "name": "Snæfellsbær",
+      "parent": "IS-3",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SOG",
+      "name": "Skeiða- og Gnúpverjahreppur",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SOL",
+      "name": "Sveitarfélagið Ölfus",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SSS",
+      "name": "Sveitarfélagið Skagaströnd",
+      "parent": "IS-5",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-STR",
+      "name": "Strandabyggð",
+      "parent": "IS-4",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-STY",
+      "name": "Stykkishólmsbær",
+      "parent": "IS-3",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-SVG",
+      "name": "Sveitarfélagið Vogar",
+      "parent": "IS-2",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-TAL",
+      "name": "Tálknafjarðarhreppur",
+      "parent": "IS-4",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-THG",
+      "name": "Þingeyjarsveit",
+      "parent": "IS-6",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-TJO",
+      "name": "Tjörneshreppur",
+      "parent": "IS-6",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-VEM",
+      "name": "Vestmannaeyjabær",
+      "parent": "IS-8",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-VER",
+      "name": "Vesturbyggð",
+      "parent": "IS-4",
+      "type": "Municipality"
+    },
+    {
+      "code": "IS-VOP",
+      "name": "Vopnafjarðarhreppur",
+      "parent": "IS-7",
+      "type": "Municipality"
+    },
+    {
+      "code": "IT-21",
+      "name": "Piemonte",
+      "type": "Region"
+    },
+    {
+      "code": "IT-23",
+      "name": "Val d'Aoste",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "IT-25",
+      "name": "Lombardia",
+      "type": "Region"
+    },
+    {
+      "code": "IT-32",
+      "name": "Trentino-Alto Adige",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "IT-34",
+      "name": "Veneto",
+      "type": "Region"
+    },
+    {
+      "code": "IT-36",
+      "name": "Friuli Venezia Giulia",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "IT-42",
+      "name": "Liguria",
+      "type": "Region"
+    },
+    {
+      "code": "IT-45",
+      "name": "Emilia-Romagna",
+      "type": "Region"
+    },
+    {
+      "code": "IT-52",
+      "name": "Toscana",
+      "type": "Region"
+    },
+    {
+      "code": "IT-55",
+      "name": "Umbria",
+      "type": "Region"
+    },
+    {
+      "code": "IT-57",
+      "name": "Marche",
+      "type": "Region"
+    },
+    {
+      "code": "IT-62",
+      "name": "Lazio",
+      "type": "Region"
+    },
+    {
+      "code": "IT-65",
+      "name": "Abruzzo",
+      "type": "Region"
+    },
+    {
+      "code": "IT-67",
+      "name": "Molise",
+      "type": "Region"
+    },
+    {
+      "code": "IT-72",
+      "name": "Campania",
+      "type": "Region"
+    },
+    {
+      "code": "IT-75",
+      "name": "Puglia",
+      "type": "Region"
+    },
+    {
+      "code": "IT-77",
+      "name": "Basilicata",
+      "type": "Region"
+    },
+    {
+      "code": "IT-78",
+      "name": "Calabria",
+      "type": "Region"
+    },
+    {
+      "code": "IT-82",
+      "name": "Sicilia",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "IT-88",
+      "name": "Sardegna",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "IT-AG",
+      "name": "Agrigento",
+      "parent": "IT-82",
+      "type": "Free municipal consortium"
+    },
+    {
+      "code": "IT-AL",
+      "name": "Alessandria",
+      "parent": "IT-21",
+      "type": "Province"
+    },
+    {
+      "code": "IT-AN",
+      "name": "Ancona",
+      "parent": "IT-57",
+      "type": "Province"
+    },
+    {
+      "code": "IT-AP",
+      "name": "Ascoli Piceno",
+      "parent": "IT-57",
+      "type": "Province"
+    },
+    {
+      "code": "IT-AQ",
+      "name": "L'Aquila",
+      "parent": "IT-65",
+      "type": "Province"
+    },
+    {
+      "code": "IT-AR",
+      "name": "Arezzo",
+      "parent": "IT-52",
+      "type": "Province"
+    },
+    {
+      "code": "IT-AT",
+      "name": "Asti",
+      "parent": "IT-21",
+      "type": "Province"
+    },
+    {
+      "code": "IT-AV",
+      "name": "Avellino",
+      "parent": "IT-72",
+      "type": "Province"
+    },
+    {
+      "code": "IT-BA",
+      "name": "Bari",
+      "parent": "IT-75",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-BG",
+      "name": "Bergamo",
+      "parent": "IT-25",
+      "type": "Province"
+    },
+    {
+      "code": "IT-BI",
+      "name": "Biella",
+      "parent": "IT-21",
+      "type": "Province"
+    },
+    {
+      "code": "IT-BL",
+      "name": "Belluno",
+      "parent": "IT-34",
+      "type": "Province"
+    },
+    {
+      "code": "IT-BN",
+      "name": "Benevento",
+      "parent": "IT-72",
+      "type": "Province"
+    },
+    {
+      "code": "IT-BO",
+      "name": "Bologna",
+      "parent": "IT-45",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-BR",
+      "name": "Brindisi",
+      "parent": "IT-75",
+      "type": "Province"
+    },
+    {
+      "code": "IT-BS",
+      "name": "Brescia",
+      "parent": "IT-25",
+      "type": "Province"
+    },
+    {
+      "code": "IT-BT",
+      "name": "Barletta-Andria-Trani",
+      "parent": "IT-75",
+      "type": "Province"
+    },
+    {
+      "code": "IT-BZ",
+      "name": "Bolzano",
+      "parent": "IT-32",
+      "type": "Autonomous province"
+    },
+    {
+      "code": "IT-CA",
+      "name": "Cagliari",
+      "parent": "IT-88",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-CB",
+      "name": "Campobasso",
+      "parent": "IT-67",
+      "type": "Province"
+    },
+    {
+      "code": "IT-CE",
+      "name": "Caserta",
+      "parent": "IT-72",
+      "type": "Province"
+    },
+    {
+      "code": "IT-CH",
+      "name": "Chieti",
+      "parent": "IT-65",
+      "type": "Province"
+    },
+    {
+      "code": "IT-CL",
+      "name": "Caltanissetta",
+      "parent": "IT-82",
+      "type": "Free municipal consortium"
+    },
+    {
+      "code": "IT-CN",
+      "name": "Cuneo",
+      "parent": "IT-21",
+      "type": "Province"
+    },
+    {
+      "code": "IT-CO",
+      "name": "Como",
+      "parent": "IT-25",
+      "type": "Province"
+    },
+    {
+      "code": "IT-CR",
+      "name": "Cremona",
+      "parent": "IT-25",
+      "type": "Province"
+    },
+    {
+      "code": "IT-CS",
+      "name": "Cosenza",
+      "parent": "IT-78",
+      "type": "Province"
+    },
+    {
+      "code": "IT-CT",
+      "name": "Catania",
+      "parent": "IT-82",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-CZ",
+      "name": "Catanzaro",
+      "parent": "IT-78",
+      "type": "Province"
+    },
+    {
+      "code": "IT-EN",
+      "name": "Enna",
+      "parent": "IT-82",
+      "type": "Free municipal consortium"
+    },
+    {
+      "code": "IT-FC",
+      "name": "Forlì-Cesena",
+      "parent": "IT-45",
+      "type": "Province"
+    },
+    {
+      "code": "IT-FE",
+      "name": "Ferrara",
+      "parent": "IT-45",
+      "type": "Province"
+    },
+    {
+      "code": "IT-FG",
+      "name": "Foggia",
+      "parent": "IT-75",
+      "type": "Province"
+    },
+    {
+      "code": "IT-FI",
+      "name": "Firenze",
+      "parent": "IT-52",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-FM",
+      "name": "Fermo",
+      "parent": "IT-57",
+      "type": "Province"
+    },
+    {
+      "code": "IT-FR",
+      "name": "Frosinone",
+      "parent": "IT-62",
+      "type": "Province"
+    },
+    {
+      "code": "IT-GE",
+      "name": "Genova",
+      "parent": "IT-42",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-GO",
+      "name": "Gorizia",
+      "parent": "IT-36",
+      "type": "Decentralized regional entity"
+    },
+    {
+      "code": "IT-GR",
+      "name": "Grosseto",
+      "parent": "IT-52",
+      "type": "Province"
+    },
+    {
+      "code": "IT-IM",
+      "name": "Imperia",
+      "parent": "IT-42",
+      "type": "Province"
+    },
+    {
+      "code": "IT-IS",
+      "name": "Isernia",
+      "parent": "IT-67",
+      "type": "Province"
+    },
+    {
+      "code": "IT-KR",
+      "name": "Crotone",
+      "parent": "IT-78",
+      "type": "Province"
+    },
+    {
+      "code": "IT-LC",
+      "name": "Lecco",
+      "parent": "IT-25",
+      "type": "Province"
+    },
+    {
+      "code": "IT-LE",
+      "name": "Lecce",
+      "parent": "IT-75",
+      "type": "Province"
+    },
+    {
+      "code": "IT-LI",
+      "name": "Livorno",
+      "parent": "IT-52",
+      "type": "Province"
+    },
+    {
+      "code": "IT-LO",
+      "name": "Lodi",
+      "parent": "IT-25",
+      "type": "Province"
+    },
+    {
+      "code": "IT-LT",
+      "name": "Latina",
+      "parent": "IT-62",
+      "type": "Province"
+    },
+    {
+      "code": "IT-LU",
+      "name": "Lucca",
+      "parent": "IT-52",
+      "type": "Province"
+    },
+    {
+      "code": "IT-MB",
+      "name": "Monza e Brianza",
+      "parent": "IT-25",
+      "type": "Province"
+    },
+    {
+      "code": "IT-MC",
+      "name": "Macerata",
+      "parent": "IT-57",
+      "type": "Province"
+    },
+    {
+      "code": "IT-ME",
+      "name": "Messina",
+      "parent": "IT-82",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-MI",
+      "name": "Milano",
+      "parent": "IT-25",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-MN",
+      "name": "Mantova",
+      "parent": "IT-25",
+      "type": "Province"
+    },
+    {
+      "code": "IT-MO",
+      "name": "Modena",
+      "parent": "IT-45",
+      "type": "Province"
+    },
+    {
+      "code": "IT-MS",
+      "name": "Massa-Carrara",
+      "parent": "IT-52",
+      "type": "Province"
+    },
+    {
+      "code": "IT-MT",
+      "name": "Matera",
+      "parent": "IT-77",
+      "type": "Province"
+    },
+    {
+      "code": "IT-NA",
+      "name": "Napoli",
+      "parent": "IT-72",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-NO",
+      "name": "Novara",
+      "parent": "IT-21",
+      "type": "Province"
+    },
+    {
+      "code": "IT-NU",
+      "name": "Nuoro",
+      "parent": "IT-88",
+      "type": "Province"
+    },
+    {
+      "code": "IT-OR",
+      "name": "Oristano",
+      "parent": "IT-88",
+      "type": "Province"
+    },
+    {
+      "code": "IT-PA",
+      "name": "Palermo",
+      "parent": "IT-82",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-PC",
+      "name": "Piacenza",
+      "parent": "IT-45",
+      "type": "Province"
+    },
+    {
+      "code": "IT-PD",
+      "name": "Padova",
+      "parent": "IT-34",
+      "type": "Province"
+    },
+    {
+      "code": "IT-PE",
+      "name": "Pescara",
+      "parent": "IT-65",
+      "type": "Province"
+    },
+    {
+      "code": "IT-PG",
+      "name": "Perugia",
+      "parent": "IT-55",
+      "type": "Province"
+    },
+    {
+      "code": "IT-PI",
+      "name": "Pisa",
+      "parent": "IT-52",
+      "type": "Province"
+    },
+    {
+      "code": "IT-PN",
+      "name": "Pordenone",
+      "parent": "IT-36",
+      "type": "Decentralized regional entity"
+    },
+    {
+      "code": "IT-PO",
+      "name": "Prato",
+      "parent": "IT-52",
+      "type": "Province"
+    },
+    {
+      "code": "IT-PR",
+      "name": "Parma",
+      "parent": "IT-45",
+      "type": "Province"
+    },
+    {
+      "code": "IT-PT",
+      "name": "Pistoia",
+      "parent": "IT-52",
+      "type": "Province"
+    },
+    {
+      "code": "IT-PU",
+      "name": "Pesaro e Urbino",
+      "parent": "IT-57",
+      "type": "Province"
+    },
+    {
+      "code": "IT-PV",
+      "name": "Pavia",
+      "parent": "IT-25",
+      "type": "Province"
+    },
+    {
+      "code": "IT-PZ",
+      "name": "Potenza",
+      "parent": "IT-77",
+      "type": "Province"
+    },
+    {
+      "code": "IT-RA",
+      "name": "Ravenna",
+      "parent": "IT-45",
+      "type": "Province"
+    },
+    {
+      "code": "IT-RC",
+      "name": "Reggio Calabria",
+      "parent": "IT-78",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-RE",
+      "name": "Reggio Emilia",
+      "parent": "IT-45",
+      "type": "Province"
+    },
+    {
+      "code": "IT-RG",
+      "name": "Ragusa",
+      "parent": "IT-82",
+      "type": "Free municipal consortium"
+    },
+    {
+      "code": "IT-RI",
+      "name": "Rieti",
+      "parent": "IT-62",
+      "type": "Province"
+    },
+    {
+      "code": "IT-RM",
+      "name": "Roma",
+      "parent": "IT-62",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-RN",
+      "name": "Rimini",
+      "parent": "IT-45",
+      "type": "Province"
+    },
+    {
+      "code": "IT-RO",
+      "name": "Rovigo",
+      "parent": "IT-34",
+      "type": "Province"
+    },
+    {
+      "code": "IT-SA",
+      "name": "Salerno",
+      "parent": "IT-72",
+      "type": "Province"
+    },
+    {
+      "code": "IT-SI",
+      "name": "Siena",
+      "parent": "IT-52",
+      "type": "Province"
+    },
+    {
+      "code": "IT-SO",
+      "name": "Sondrio",
+      "parent": "IT-25",
+      "type": "Province"
+    },
+    {
+      "code": "IT-SP",
+      "name": "La Spezia",
+      "parent": "IT-42",
+      "type": "Province"
+    },
+    {
+      "code": "IT-SR",
+      "name": "Siracusa",
+      "parent": "IT-82",
+      "type": "Free municipal consortium"
+    },
+    {
+      "code": "IT-SS",
+      "name": "Sassari",
+      "parent": "IT-88",
+      "type": "Province"
+    },
+    {
+      "code": "IT-SU",
+      "name": "Sud Sardegna",
+      "parent": "IT-88",
+      "type": "Province"
+    },
+    {
+      "code": "IT-SV",
+      "name": "Savona",
+      "parent": "IT-42",
+      "type": "Province"
+    },
+    {
+      "code": "IT-TA",
+      "name": "Taranto",
+      "parent": "IT-75",
+      "type": "Province"
+    },
+    {
+      "code": "IT-TE",
+      "name": "Teramo",
+      "parent": "IT-65",
+      "type": "Province"
+    },
+    {
+      "code": "IT-TN",
+      "name": "Trento",
+      "parent": "IT-32",
+      "type": "Autonomous province"
+    },
+    {
+      "code": "IT-TO",
+      "name": "Torino",
+      "parent": "IT-21",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-TP",
+      "name": "Trapani",
+      "parent": "IT-82",
+      "type": "Free municipal consortium"
+    },
+    {
+      "code": "IT-TR",
+      "name": "Terni",
+      "parent": "IT-55",
+      "type": "Province"
+    },
+    {
+      "code": "IT-TS",
+      "name": "Trieste",
+      "parent": "IT-36",
+      "type": "Decentralized regional entity"
+    },
+    {
+      "code": "IT-TV",
+      "name": "Treviso",
+      "parent": "IT-34",
+      "type": "Province"
+    },
+    {
+      "code": "IT-UD",
+      "name": "Udine",
+      "parent": "IT-36",
+      "type": "Decentralized regional entity"
+    },
+    {
+      "code": "IT-VA",
+      "name": "Varese",
+      "parent": "IT-25",
+      "type": "Province"
+    },
+    {
+      "code": "IT-VB",
+      "name": "Verbano-Cusio-Ossola",
+      "parent": "IT-21",
+      "type": "Province"
+    },
+    {
+      "code": "IT-VC",
+      "name": "Vercelli",
+      "parent": "IT-21",
+      "type": "Province"
+    },
+    {
+      "code": "IT-VE",
+      "name": "Venezia",
+      "parent": "IT-34",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "IT-VI",
+      "name": "Vicenza",
+      "parent": "IT-34",
+      "type": "Province"
+    },
+    {
+      "code": "IT-VR",
+      "name": "Verona",
+      "parent": "IT-34",
+      "type": "Province"
+    },
+    {
+      "code": "IT-VT",
+      "name": "Viterbo",
+      "parent": "IT-62",
+      "type": "Province"
+    },
+    {
+      "code": "IT-VV",
+      "name": "Vibo Valentia",
+      "parent": "IT-78",
+      "type": "Province"
+    },
+    {
+      "code": "JM-01",
+      "name": "Kingston",
+      "type": "Parish"
+    },
+    {
+      "code": "JM-02",
+      "name": "Saint Andrew",
+      "type": "Parish"
+    },
+    {
+      "code": "JM-03",
+      "name": "Saint Thomas",
+      "type": "Parish"
+    },
+    {
+      "code": "JM-04",
+      "name": "Portland",
+      "type": "Parish"
+    },
+    {
+      "code": "JM-05",
+      "name": "Saint Mary",
+      "type": "Parish"
+    },
+    {
+      "code": "JM-06",
+      "name": "Saint Ann",
+      "type": "Parish"
+    },
+    {
+      "code": "JM-07",
+      "name": "Trelawny",
+      "type": "Parish"
+    },
+    {
+      "code": "JM-08",
+      "name": "Saint James",
+      "type": "Parish"
+    },
+    {
+      "code": "JM-09",
+      "name": "Hanover",
+      "type": "Parish"
+    },
+    {
+      "code": "JM-10",
+      "name": "Westmoreland",
+      "type": "Parish"
+    },
+    {
+      "code": "JM-11",
+      "name": "Saint Elizabeth",
+      "type": "Parish"
+    },
+    {
+      "code": "JM-12",
+      "name": "Manchester",
+      "type": "Parish"
+    },
+    {
+      "code": "JM-13",
+      "name": "Clarendon",
+      "type": "Parish"
+    },
+    {
+      "code": "JM-14",
+      "name": "Saint Catherine",
+      "type": "Parish"
+    },
+    {
+      "code": "JO-AJ",
+      "name": "‘Ajlūn",
+      "type": "Governorate"
+    },
+    {
+      "code": "JO-AM",
+      "name": "Al ‘A̅şimah",
+      "type": "Governorate"
+    },
+    {
+      "code": "JO-AQ",
+      "name": "Al ‘Aqabah",
+      "type": "Governorate"
+    },
+    {
+      "code": "JO-AT",
+      "name": "Aţ Ţafīlah",
+      "type": "Governorate"
+    },
+    {
+      "code": "JO-AZ",
+      "name": "Az Zarqā’",
+      "type": "Governorate"
+    },
+    {
+      "code": "JO-BA",
+      "name": "Al Balqā’",
+      "type": "Governorate"
+    },
+    {
+      "code": "JO-IR",
+      "name": "Irbid",
+      "type": "Governorate"
+    },
+    {
+      "code": "JO-JA",
+      "name": "Jarash",
+      "type": "Governorate"
+    },
+    {
+      "code": "JO-KA",
+      "name": "Al Karak",
+      "type": "Governorate"
+    },
+    {
+      "code": "JO-MA",
+      "name": "Al Mafraq",
+      "type": "Governorate"
+    },
+    {
+      "code": "JO-MD",
+      "name": "Mādabā",
+      "type": "Governorate"
+    },
+    {
+      "code": "JO-MN",
+      "name": "Ma‘ān",
+      "type": "Governorate"
+    },
+    {
+      "code": "JP-01",
+      "name": "Hokkaido",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-02",
+      "name": "Aomori",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-03",
+      "name": "Iwate",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-04",
+      "name": "Miyagi",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-05",
+      "name": "Akita",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-06",
+      "name": "Yamagata",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-07",
+      "name": "Fukushima",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-08",
+      "name": "Ibaraki",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-09",
+      "name": "Tochigi",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-10",
+      "name": "Gunma",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-11",
+      "name": "Saitama",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-12",
+      "name": "Chiba",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-13",
+      "name": "Tokyo",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-14",
+      "name": "Kanagawa",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-15",
+      "name": "Niigata",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-16",
+      "name": "Toyama",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-17",
+      "name": "Ishikawa",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-18",
+      "name": "Fukui",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-19",
+      "name": "Yamanashi",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-20",
+      "name": "Nagano",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-21",
+      "name": "Gifu",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-22",
+      "name": "Shizuoka",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-23",
+      "name": "Aichi",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-24",
+      "name": "Mie",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-25",
+      "name": "Shiga",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-26",
+      "name": "Kyoto",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-27",
+      "name": "Osaka",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-28",
+      "name": "Hyogo",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-29",
+      "name": "Nara",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-30",
+      "name": "Wakayama",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-31",
+      "name": "Tottori",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-32",
+      "name": "Shimane",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-33",
+      "name": "Okayama",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-34",
+      "name": "Hiroshima",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-35",
+      "name": "Yamaguchi",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-36",
+      "name": "Tokushima",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-37",
+      "name": "Kagawa",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-38",
+      "name": "Ehime",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-39",
+      "name": "Kochi",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-40",
+      "name": "Fukuoka",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-41",
+      "name": "Saga",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-42",
+      "name": "Nagasaki",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-43",
+      "name": "Kumamoto",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-44",
+      "name": "Oita",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-45",
+      "name": "Miyazaki",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-46",
+      "name": "Kagoshima",
+      "type": "Prefecture"
+    },
+    {
+      "code": "JP-47",
+      "name": "Okinawa",
+      "type": "Prefecture"
+    },
+    {
+      "code": "KE-01",
+      "name": "Baringo",
+      "type": "County"
+    },
+    {
+      "code": "KE-02",
+      "name": "Bomet",
+      "type": "County"
+    },
+    {
+      "code": "KE-03",
+      "name": "Bungoma",
+      "type": "County"
+    },
+    {
+      "code": "KE-04",
+      "name": "Busia",
+      "type": "County"
+    },
+    {
+      "code": "KE-05",
+      "name": "Elgeyo/Marakwet",
+      "type": "County"
+    },
+    {
+      "code": "KE-06",
+      "name": "Embu",
+      "type": "County"
+    },
+    {
+      "code": "KE-07",
+      "name": "Garissa",
+      "type": "County"
+    },
+    {
+      "code": "KE-08",
+      "name": "Homa Bay",
+      "type": "County"
+    },
+    {
+      "code": "KE-09",
+      "name": "Isiolo",
+      "type": "County"
+    },
+    {
+      "code": "KE-10",
+      "name": "Kajiado",
+      "type": "County"
+    },
+    {
+      "code": "KE-11",
+      "name": "Kakamega",
+      "type": "County"
+    },
+    {
+      "code": "KE-12",
+      "name": "Kericho",
+      "type": "County"
+    },
+    {
+      "code": "KE-13",
+      "name": "Kiambu",
+      "type": "County"
+    },
+    {
+      "code": "KE-14",
+      "name": "Kilifi",
+      "type": "County"
+    },
+    {
+      "code": "KE-15",
+      "name": "Kirinyaga",
+      "type": "County"
+    },
+    {
+      "code": "KE-16",
+      "name": "Kisii",
+      "type": "County"
+    },
+    {
+      "code": "KE-17",
+      "name": "Kisumu",
+      "type": "County"
+    },
+    {
+      "code": "KE-18",
+      "name": "Kitui",
+      "type": "County"
+    },
+    {
+      "code": "KE-19",
+      "name": "Kwale",
+      "type": "County"
+    },
+    {
+      "code": "KE-20",
+      "name": "Laikipia",
+      "type": "County"
+    },
+    {
+      "code": "KE-21",
+      "name": "Lamu",
+      "type": "County"
+    },
+    {
+      "code": "KE-22",
+      "name": "Machakos",
+      "type": "County"
+    },
+    {
+      "code": "KE-23",
+      "name": "Makueni",
+      "type": "County"
+    },
+    {
+      "code": "KE-24",
+      "name": "Mandera",
+      "type": "County"
+    },
+    {
+      "code": "KE-25",
+      "name": "Marsabit",
+      "type": "County"
+    },
+    {
+      "code": "KE-26",
+      "name": "Meru",
+      "type": "County"
+    },
+    {
+      "code": "KE-27",
+      "name": "Migori",
+      "type": "County"
+    },
+    {
+      "code": "KE-28",
+      "name": "Mombasa",
+      "type": "County"
+    },
+    {
+      "code": "KE-29",
+      "name": "Murang'a",
+      "type": "County"
+    },
+    {
+      "code": "KE-30",
+      "name": "Nairobi City",
+      "type": "County"
+    },
+    {
+      "code": "KE-31",
+      "name": "Nakuru",
+      "type": "County"
+    },
+    {
+      "code": "KE-32",
+      "name": "Nandi",
+      "type": "County"
+    },
+    {
+      "code": "KE-33",
+      "name": "Narok",
+      "type": "County"
+    },
+    {
+      "code": "KE-34",
+      "name": "Nyamira",
+      "type": "County"
+    },
+    {
+      "code": "KE-35",
+      "name": "Nyandarua",
+      "type": "County"
+    },
+    {
+      "code": "KE-36",
+      "name": "Nyeri",
+      "type": "County"
+    },
+    {
+      "code": "KE-37",
+      "name": "Samburu",
+      "type": "County"
+    },
+    {
+      "code": "KE-38",
+      "name": "Siaya",
+      "type": "County"
+    },
+    {
+      "code": "KE-39",
+      "name": "Taita/Taveta",
+      "type": "County"
+    },
+    {
+      "code": "KE-40",
+      "name": "Tana River",
+      "type": "County"
+    },
+    {
+      "code": "KE-41",
+      "name": "Tharaka-Nithi",
+      "type": "County"
+    },
+    {
+      "code": "KE-42",
+      "name": "Trans Nzoia",
+      "type": "County"
+    },
+    {
+      "code": "KE-43",
+      "name": "Turkana",
+      "type": "County"
+    },
+    {
+      "code": "KE-44",
+      "name": "Uasin Gishu",
+      "type": "County"
+    },
+    {
+      "code": "KE-45",
+      "name": "Vihiga",
+      "type": "County"
+    },
+    {
+      "code": "KE-46",
+      "name": "Wajir",
+      "type": "County"
+    },
+    {
+      "code": "KE-47",
+      "name": "West Pokot",
+      "type": "County"
+    },
+    {
+      "code": "KG-B",
+      "name": "Batken",
+      "type": "Region"
+    },
+    {
+      "code": "KG-C",
+      "name": "Chuyskaya oblast'",
+      "type": "Region"
+    },
+    {
+      "code": "KG-GB",
+      "name": "Bishkek Shaary",
+      "type": "City"
+    },
+    {
+      "code": "KG-GO",
+      "name": "Gorod Osh",
+      "type": "City"
+    },
+    {
+      "code": "KG-J",
+      "name": "Dzhalal-Abadskaya oblast'",
+      "type": "Region"
+    },
+    {
+      "code": "KG-N",
+      "name": "Naryn",
+      "type": "Region"
+    },
+    {
+      "code": "KG-O",
+      "name": "Osh",
+      "type": "Region"
+    },
+    {
+      "code": "KG-T",
+      "name": "Talas",
+      "type": "Region"
+    },
+    {
+      "code": "KG-Y",
+      "name": "Issyk-Kul'skaja oblast'",
+      "type": "Region"
+    },
+    {
+      "code": "KH-1",
+      "name": "Banteay Mean Choăy",
+      "type": "Province"
+    },
+    {
+      "code": "KH-10",
+      "name": "Kracheh",
+      "type": "Province"
+    },
+    {
+      "code": "KH-11",
+      "name": "Mondol Kiri",
+      "type": "Province"
+    },
+    {
+      "code": "KH-12",
+      "name": "Phnom Penh",
+      "type": "Autonomous municipality"
+    },
+    {
+      "code": "KH-13",
+      "name": "Preah Vihear",
+      "type": "Province"
+    },
+    {
+      "code": "KH-14",
+      "name": "Prey Veaeng",
+      "type": "Province"
+    },
+    {
+      "code": "KH-15",
+      "name": "Pousaat",
+      "type": "Province"
+    },
+    {
+      "code": "KH-16",
+      "name": "Rotanak Kiri",
+      "type": "Province"
+    },
+    {
+      "code": "KH-17",
+      "name": "Siem Reab",
+      "type": "Province"
+    },
+    {
+      "code": "KH-18",
+      "name": "Preah Sihanouk",
+      "type": "Province"
+    },
+    {
+      "code": "KH-19",
+      "name": "Stoĕng Trêng",
+      "type": "Province"
+    },
+    {
+      "code": "KH-2",
+      "name": "Baat Dambang",
+      "type": "Province"
+    },
+    {
+      "code": "KH-20",
+      "name": "Svaay Rieng",
+      "type": "Province"
+    },
+    {
+      "code": "KH-21",
+      "name": "Taakaev",
+      "type": "Province"
+    },
+    {
+      "code": "KH-22",
+      "name": "Otdar Mean Chey",
+      "type": "Province"
+    },
+    {
+      "code": "KH-23",
+      "name": "Kaeb",
+      "type": "Province"
+    },
+    {
+      "code": "KH-24",
+      "name": "Pailin",
+      "type": "Province"
+    },
+    {
+      "code": "KH-25",
+      "name": "Tbong Khmum",
+      "type": "Province"
+    },
+    {
+      "code": "KH-3",
+      "name": "Kampong Chaam",
+      "type": "Province"
+    },
+    {
+      "code": "KH-4",
+      "name": "Kampong Chhnang",
+      "type": "Province"
+    },
+    {
+      "code": "KH-5",
+      "name": "Kampong Spueu",
+      "type": "Province"
+    },
+    {
+      "code": "KH-6",
+      "name": "Kampong Thum",
+      "type": "Province"
+    },
+    {
+      "code": "KH-7",
+      "name": "Kampot",
+      "type": "Province"
+    },
+    {
+      "code": "KH-8",
+      "name": "Kandaal",
+      "type": "Province"
+    },
+    {
+      "code": "KH-9",
+      "name": "Kaoh Kong",
+      "type": "Province"
+    },
+    {
+      "code": "KI-G",
+      "name": "Gilbert Islands",
+      "type": "Group of islands (20 inhabited islands)"
+    },
+    {
+      "code": "KI-L",
+      "name": "Line Islands",
+      "type": "Group of islands (20 inhabited islands)"
+    },
+    {
+      "code": "KI-P",
+      "name": "Phoenix Islands",
+      "type": "Group of islands (20 inhabited islands)"
+    },
+    {
+      "code": "KM-A",
+      "name": "Anjouan",
+      "type": "Island"
+    },
+    {
+      "code": "KM-G",
+      "name": "Grande Comore",
+      "type": "Island"
+    },
+    {
+      "code": "KM-M",
+      "name": "Mohéli",
+      "type": "Island"
+    },
+    {
+      "code": "KN-01",
+      "name": "Christ Church Nichola Town",
+      "parent": "KN-K",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-02",
+      "name": "Saint Anne Sandy Point",
+      "parent": "KN-K",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-03",
+      "name": "Saint George Basseterre",
+      "parent": "KN-K",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-04",
+      "name": "Saint George Gingerland",
+      "parent": "KN-N",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-05",
+      "name": "Saint James Windward",
+      "parent": "KN-N",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-06",
+      "name": "Saint John Capisterre",
+      "parent": "KN-K",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-07",
+      "name": "Saint John Figtree",
+      "parent": "KN-N",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-08",
+      "name": "Saint Mary Cayon",
+      "parent": "KN-K",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-09",
+      "name": "Saint Paul Capisterre",
+      "parent": "KN-K",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-10",
+      "name": "Saint Paul Charlestown",
+      "parent": "KN-N",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-11",
+      "name": "Saint Peter Basseterre",
+      "parent": "KN-K",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-12",
+      "name": "Saint Thomas Lowland",
+      "parent": "KN-N",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-13",
+      "name": "Saint Thomas Middle Island",
+      "parent": "KN-K",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-15",
+      "name": "Trinity Palmetto Point",
+      "parent": "KN-K",
+      "type": "Parish"
+    },
+    {
+      "code": "KN-K",
+      "name": "Saint Kitts",
+      "type": "State"
+    },
+    {
+      "code": "KN-N",
+      "name": "Nevis",
+      "type": "State"
+    },
+    {
+      "code": "KP-01",
+      "name": "P'yǒngyang",
+      "type": "Capital city"
+    },
+    {
+      "code": "KP-02",
+      "name": "P'yǒngan-namdo",
+      "type": "Province"
+    },
+    {
+      "code": "KP-03",
+      "name": "P'yǒngan-bukto",
+      "type": "Province"
+    },
+    {
+      "code": "KP-04",
+      "name": "Chagang-do",
+      "type": "Province"
+    },
+    {
+      "code": "KP-05",
+      "name": "Hwanghae-namdo",
+      "type": "Province"
+    },
+    {
+      "code": "KP-06",
+      "name": "Hwanghae-bukto",
+      "type": "Province"
+    },
+    {
+      "code": "KP-07",
+      "name": "Kangweonto",
+      "type": "Province"
+    },
+    {
+      "code": "KP-08",
+      "name": "Hamgyǒng-namdo",
+      "type": "Province"
+    },
+    {
+      "code": "KP-09",
+      "name": "Hamgyǒng-bukto",
+      "type": "Province"
+    },
+    {
+      "code": "KP-10",
+      "name": "Ryanggang-do",
+      "type": "Province"
+    },
+    {
+      "code": "KP-13",
+      "name": "Raseon",
+      "type": "Special city"
+    },
+    {
+      "code": "KP-14",
+      "name": "Nampho",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "KP-15",
+      "name": "Kaeseong",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "KR-11",
+      "name": "Seoul-teukbyeolsi",
+      "type": "Special city"
+    },
+    {
+      "code": "KR-26",
+      "name": "Busan-gwangyeoksi",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "KR-27",
+      "name": "Daegu-gwangyeoksi",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "KR-28",
+      "name": "Incheon-gwangyeoksi",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "KR-29",
+      "name": "Gwangju-gwangyeoksi",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "KR-30",
+      "name": "Daejeon-gwangyeoksi",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "KR-31",
+      "name": "Ulsan-gwangyeoksi",
+      "type": "Metropolitan city"
+    },
+    {
+      "code": "KR-41",
+      "name": "Gyeonggi-do",
+      "type": "Province"
+    },
+    {
+      "code": "KR-42",
+      "name": "Gangwon-teukbyeoljachido",
+      "type": "Special self-governing province"
+    },
+    {
+      "code": "KR-43",
+      "name": "Chungcheongbuk-do",
+      "type": "Province"
+    },
+    {
+      "code": "KR-44",
+      "name": "Chungcheongnam-do",
+      "type": "Province"
+    },
+    {
+      "code": "KR-45",
+      "name": "Jeollabuk-do",
+      "type": "Province"
+    },
+    {
+      "code": "KR-46",
+      "name": "Jeollanam-do",
+      "type": "Province"
+    },
+    {
+      "code": "KR-47",
+      "name": "Gyeongsangbuk-do",
+      "type": "Province"
+    },
+    {
+      "code": "KR-48",
+      "name": "Gyeongsangnam-do",
+      "type": "Province"
+    },
+    {
+      "code": "KR-49",
+      "name": "Jeju-teukbyeoljachido",
+      "type": "Special self-governing province"
+    },
+    {
+      "code": "KR-50",
+      "name": "Sejong",
+      "type": "Special self-governing city"
+    },
+    {
+      "code": "KW-AH",
+      "name": "Al Aḩmadī",
+      "type": "Governorate"
+    },
+    {
+      "code": "KW-FA",
+      "name": "Al Farwānīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "KW-HA",
+      "name": "Ḩawallī",
+      "type": "Governorate"
+    },
+    {
+      "code": "KW-JA",
+      "name": "Al Jahrā’",
+      "type": "Governorate"
+    },
+    {
+      "code": "KW-KU",
+      "name": "Al ‘Āşimah",
+      "type": "Governorate"
+    },
+    {
+      "code": "KW-MU",
+      "name": "Mubārak al Kabīr",
+      "type": "Governorate"
+    },
+    {
+      "code": "KZ-10",
+      "name": "Abajskaja oblast’",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-11",
+      "name": "Akmolinskaja oblast'",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-15",
+      "name": "Aktjubinskaja oblast'",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-19",
+      "name": "Almatinskaja oblast'",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-23",
+      "name": "Atyrauskaja oblast'",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-27",
+      "name": "Batys Qazaqstan oblysy",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-31",
+      "name": "Zhambyl oblysy",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-33",
+      "name": "Zhetisū oblysy",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-35",
+      "name": "Karagandinskaja oblast'",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-39",
+      "name": "Kostanajskaja oblast'",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-43",
+      "name": "Kyzylordinskaja oblast'",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-47",
+      "name": "Mangghystaū oblysy",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-55",
+      "name": "Pavlodar oblysy",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-59",
+      "name": "Severo-Kazahstanskaja oblast'",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-61",
+      "name": "Turkestankaya oblast'",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-62",
+      "name": "Ulytauskaja oblast’",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-63",
+      "name": "Shyghys Qazaqstan oblysy",
+      "type": "Region"
+    },
+    {
+      "code": "KZ-71",
+      "name": "Astana",
+      "type": "City"
+    },
+    {
+      "code": "KZ-75",
+      "name": "Almaty",
+      "type": "City"
+    },
+    {
+      "code": "KZ-79",
+      "name": "Shymkent",
+      "type": "City"
+    },
+    {
+      "code": "LA-AT",
+      "name": "Attapu",
+      "type": "Province"
+    },
+    {
+      "code": "LA-BK",
+      "name": "Bokèo",
+      "type": "Province"
+    },
+    {
+      "code": "LA-BL",
+      "name": "Bolikhamxai",
+      "type": "Province"
+    },
+    {
+      "code": "LA-CH",
+      "name": "Champasak",
+      "type": "Province"
+    },
+    {
+      "code": "LA-HO",
+      "name": "Houaphan",
+      "type": "Province"
+    },
+    {
+      "code": "LA-KH",
+      "name": "Khammouan",
+      "type": "Province"
+    },
+    {
+      "code": "LA-LM",
+      "name": "Louang Namtha",
+      "type": "Province"
+    },
+    {
+      "code": "LA-LP",
+      "name": "Louangphabang",
+      "type": "Province"
+    },
+    {
+      "code": "LA-OU",
+      "name": "Oudômxai",
+      "type": "Province"
+    },
+    {
+      "code": "LA-PH",
+      "name": "Phôngsali",
+      "type": "Province"
+    },
+    {
+      "code": "LA-SL",
+      "name": "Salavan",
+      "type": "Province"
+    },
+    {
+      "code": "LA-SV",
+      "name": "Savannakhét",
+      "type": "Province"
+    },
+    {
+      "code": "LA-VI",
+      "name": "Viangchan",
+      "type": "Province"
+    },
+    {
+      "code": "LA-VT",
+      "name": "Viangchan",
+      "type": "Prefecture"
+    },
+    {
+      "code": "LA-XA",
+      "name": "Xaignabouli",
+      "type": "Province"
+    },
+    {
+      "code": "LA-XE",
+      "name": "Xékong",
+      "type": "Province"
+    },
+    {
+      "code": "LA-XI",
+      "name": "Xiangkhouang",
+      "type": "Province"
+    },
+    {
+      "code": "LA-XS",
+      "name": "Xaisômboun",
+      "type": "Province"
+    },
+    {
+      "code": "LB-AK",
+      "name": "Aakkâr",
+      "type": "Governorate"
+    },
+    {
+      "code": "LB-AS",
+      "name": "Ash Shimāl",
+      "type": "Governorate"
+    },
+    {
+      "code": "LB-BA",
+      "name": "Bayrūt",
+      "type": "Governorate"
+    },
+    {
+      "code": "LB-BH",
+      "name": "Baalbek-Hermel",
+      "type": "Governorate"
+    },
+    {
+      "code": "LB-BI",
+      "name": "Al Biqā‘",
+      "type": "Governorate"
+    },
+    {
+      "code": "LB-JA",
+      "name": "Al Janūb",
+      "type": "Governorate"
+    },
+    {
+      "code": "LB-JL",
+      "name": "Jabal Lubnān",
+      "type": "Governorate"
+    },
+    {
+      "code": "LB-NA",
+      "name": "An Nabaţīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "LC-01",
+      "name": "Anse la Raye",
+      "type": "District"
+    },
+    {
+      "code": "LC-02",
+      "name": "Castries",
+      "type": "District"
+    },
+    {
+      "code": "LC-03",
+      "name": "Choiseul",
+      "type": "District"
+    },
+    {
+      "code": "LC-05",
+      "name": "Dennery",
+      "type": "District"
+    },
+    {
+      "code": "LC-06",
+      "name": "Gros Islet",
+      "type": "District"
+    },
+    {
+      "code": "LC-07",
+      "name": "Laborie",
+      "type": "District"
+    },
+    {
+      "code": "LC-08",
+      "name": "Micoud",
+      "type": "District"
+    },
+    {
+      "code": "LC-10",
+      "name": "Soufrière",
+      "type": "District"
+    },
+    {
+      "code": "LC-11",
+      "name": "Vieux Fort",
+      "type": "District"
+    },
+    {
+      "code": "LC-12",
+      "name": "Canaries",
+      "type": "District"
+    },
+    {
+      "code": "LI-01",
+      "name": "Balzers",
+      "type": "Commune"
+    },
+    {
+      "code": "LI-02",
+      "name": "Eschen",
+      "type": "Commune"
+    },
+    {
+      "code": "LI-03",
+      "name": "Gamprin",
+      "type": "Commune"
+    },
+    {
+      "code": "LI-04",
+      "name": "Mauren",
+      "type": "Commune"
+    },
+    {
+      "code": "LI-05",
+      "name": "Planken",
+      "type": "Commune"
+    },
+    {
+      "code": "LI-06",
+      "name": "Ruggell",
+      "type": "Commune"
+    },
+    {
+      "code": "LI-07",
+      "name": "Schaan",
+      "type": "Commune"
+    },
+    {
+      "code": "LI-08",
+      "name": "Schellenberg",
+      "type": "Commune"
+    },
+    {
+      "code": "LI-09",
+      "name": "Triesen",
+      "type": "Commune"
+    },
+    {
+      "code": "LI-10",
+      "name": "Triesenberg",
+      "type": "Commune"
+    },
+    {
+      "code": "LI-11",
+      "name": "Vaduz",
+      "type": "Commune"
+    },
+    {
+      "code": "LK-1",
+      "name": "Western Province",
+      "type": "Province"
+    },
+    {
+      "code": "LK-11",
+      "name": "Colombo",
+      "parent": "LK-1",
+      "type": "District"
+    },
+    {
+      "code": "LK-12",
+      "name": "Gampaha",
+      "parent": "LK-1",
+      "type": "District"
+    },
+    {
+      "code": "LK-13",
+      "name": "Kalutara",
+      "parent": "LK-1",
+      "type": "District"
+    },
+    {
+      "code": "LK-2",
+      "name": "Central Province",
+      "type": "Province"
+    },
+    {
+      "code": "LK-21",
+      "name": "Kandy",
+      "parent": "LK-2",
+      "type": "District"
+    },
+    {
+      "code": "LK-22",
+      "name": "Matale",
+      "parent": "LK-2",
+      "type": "District"
+    },
+    {
+      "code": "LK-23",
+      "name": "Nuwara Eliya",
+      "parent": "LK-2",
+      "type": "District"
+    },
+    {
+      "code": "LK-3",
+      "name": "Southern Province",
+      "type": "Province"
+    },
+    {
+      "code": "LK-31",
+      "name": "Galle",
+      "parent": "LK-3",
+      "type": "District"
+    },
+    {
+      "code": "LK-32",
+      "name": "Matara",
+      "parent": "LK-3",
+      "type": "District"
+    },
+    {
+      "code": "LK-33",
+      "name": "Hambantota",
+      "parent": "LK-3",
+      "type": "District"
+    },
+    {
+      "code": "LK-4",
+      "name": "Northern Province",
+      "type": "Province"
+    },
+    {
+      "code": "LK-41",
+      "name": "Jaffna",
+      "parent": "LK-4",
+      "type": "District"
+    },
+    {
+      "code": "LK-42",
+      "name": "Kilinochchi",
+      "parent": "LK-4",
+      "type": "District"
+    },
+    {
+      "code": "LK-43",
+      "name": "Mannar",
+      "parent": "LK-4",
+      "type": "District"
+    },
+    {
+      "code": "LK-44",
+      "name": "Vavuniya",
+      "parent": "LK-4",
+      "type": "District"
+    },
+    {
+      "code": "LK-45",
+      "name": "Mullaittivu",
+      "parent": "LK-4",
+      "type": "District"
+    },
+    {
+      "code": "LK-5",
+      "name": "Eastern Province",
+      "type": "Province"
+    },
+    {
+      "code": "LK-51",
+      "name": "Batticaloa",
+      "parent": "LK-5",
+      "type": "District"
+    },
+    {
+      "code": "LK-52",
+      "name": "Ampara",
+      "parent": "LK-5",
+      "type": "District"
+    },
+    {
+      "code": "LK-53",
+      "name": "Trincomalee",
+      "parent": "LK-5",
+      "type": "District"
+    },
+    {
+      "code": "LK-6",
+      "name": "North Western Province",
+      "type": "Province"
+    },
+    {
+      "code": "LK-61",
+      "name": "Kurunegala",
+      "parent": "LK-6",
+      "type": "District"
+    },
+    {
+      "code": "LK-62",
+      "name": "Puttalam",
+      "parent": "LK-6",
+      "type": "District"
+    },
+    {
+      "code": "LK-7",
+      "name": "North Central Province",
+      "type": "Province"
+    },
+    {
+      "code": "LK-71",
+      "name": "Anuradhapura",
+      "parent": "LK-7",
+      "type": "District"
+    },
+    {
+      "code": "LK-72",
+      "name": "Polonnaruwa",
+      "parent": "LK-7",
+      "type": "District"
+    },
+    {
+      "code": "LK-8",
+      "name": "Uva Province",
+      "type": "Province"
+    },
+    {
+      "code": "LK-81",
+      "name": "Badulla",
+      "parent": "LK-8",
+      "type": "District"
+    },
+    {
+      "code": "LK-82",
+      "name": "Monaragala",
+      "parent": "LK-8",
+      "type": "District"
+    },
+    {
+      "code": "LK-9",
+      "name": "Sabaragamuwa Province",
+      "type": "Province"
+    },
+    {
+      "code": "LK-91",
+      "name": "Ratnapura",
+      "parent": "LK-9",
+      "type": "District"
+    },
+    {
+      "code": "LK-92",
+      "name": "Kegalla",
+      "parent": "LK-9",
+      "type": "District"
+    },
+    {
+      "code": "LR-BG",
+      "name": "Bong",
+      "type": "County"
+    },
+    {
+      "code": "LR-BM",
+      "name": "Bomi",
+      "type": "County"
+    },
+    {
+      "code": "LR-CM",
+      "name": "Grand Cape Mount",
+      "type": "County"
+    },
+    {
+      "code": "LR-GB",
+      "name": "Grand Bassa",
+      "type": "County"
+    },
+    {
+      "code": "LR-GG",
+      "name": "Grand Gedeh",
+      "type": "County"
+    },
+    {
+      "code": "LR-GK",
+      "name": "Grand Kru",
+      "type": "County"
+    },
+    {
+      "code": "LR-GP",
+      "name": "Gbarpolu",
+      "type": "County"
+    },
+    {
+      "code": "LR-LO",
+      "name": "Lofa",
+      "type": "County"
+    },
+    {
+      "code": "LR-MG",
+      "name": "Margibi",
+      "type": "County"
+    },
+    {
+      "code": "LR-MO",
+      "name": "Montserrado",
+      "type": "County"
+    },
+    {
+      "code": "LR-MY",
+      "name": "Maryland",
+      "type": "County"
+    },
+    {
+      "code": "LR-NI",
+      "name": "Nimba",
+      "type": "County"
+    },
+    {
+      "code": "LR-RG",
+      "name": "River Gee",
+      "type": "County"
+    },
+    {
+      "code": "LR-RI",
+      "name": "River Cess",
+      "type": "County"
+    },
+    {
+      "code": "LR-SI",
+      "name": "Sinoe",
+      "type": "County"
+    },
+    {
+      "code": "LS-A",
+      "name": "Maseru",
+      "type": "District"
+    },
+    {
+      "code": "LS-B",
+      "name": "Botha-Bothe",
+      "type": "District"
+    },
+    {
+      "code": "LS-C",
+      "name": "Leribe",
+      "type": "District"
+    },
+    {
+      "code": "LS-D",
+      "name": "Berea",
+      "type": "District"
+    },
+    {
+      "code": "LS-E",
+      "name": "Mafeteng",
+      "type": "District"
+    },
+    {
+      "code": "LS-F",
+      "name": "Mohale's Hoek",
+      "type": "District"
+    },
+    {
+      "code": "LS-G",
+      "name": "Quthing",
+      "type": "District"
+    },
+    {
+      "code": "LS-H",
+      "name": "Qacha's Nek",
+      "type": "District"
+    },
+    {
+      "code": "LS-J",
+      "name": "Mokhotlong",
+      "type": "District"
+    },
+    {
+      "code": "LS-K",
+      "name": "Thaba-Tseka",
+      "type": "District"
+    },
+    {
+      "code": "LT-01",
+      "name": "Akmenė",
+      "parent": "LT-SA",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-02",
+      "name": "Alytaus miestas",
+      "parent": "LT-AL",
+      "type": "City municipality"
+    },
+    {
+      "code": "LT-03",
+      "name": "Alytus",
+      "parent": "LT-AL",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-04",
+      "name": "Anykščiai",
+      "parent": "LT-UT",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-05",
+      "name": "Birštonas",
+      "parent": "LT-KU",
+      "type": "Municipality"
+    },
+    {
+      "code": "LT-06",
+      "name": "Biržai",
+      "parent": "LT-PN",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-07",
+      "name": "Druskininkai",
+      "parent": "LT-AL",
+      "type": "Municipality"
+    },
+    {
+      "code": "LT-08",
+      "name": "Elektrėnai",
+      "parent": "LT-VL",
+      "type": "Municipality"
+    },
+    {
+      "code": "LT-09",
+      "name": "Ignalina",
+      "parent": "LT-UT",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-10",
+      "name": "Jonava",
+      "parent": "LT-KU",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-11",
+      "name": "Joniškis",
+      "parent": "LT-SA",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-12",
+      "name": "Jurbarkas",
+      "parent": "LT-TA",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-13",
+      "name": "Kaišiadorys",
+      "parent": "LT-KU",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-14",
+      "name": "Kalvarija",
+      "parent": "LT-MR",
+      "type": "Municipality"
+    },
+    {
+      "code": "LT-15",
+      "name": "Kauno miestas",
+      "parent": "LT-KU",
+      "type": "City municipality"
+    },
+    {
+      "code": "LT-16",
+      "name": "Kaunas",
+      "parent": "LT-KU",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-17",
+      "name": "Kazlų Rūdos",
+      "parent": "LT-MR",
+      "type": "Municipality"
+    },
+    {
+      "code": "LT-18",
+      "name": "Kėdainiai",
+      "parent": "LT-KU",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-19",
+      "name": "Kelmė",
+      "parent": "LT-SA",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-20",
+      "name": "Klaipėdos miestas",
+      "parent": "LT-KL",
+      "type": "City municipality"
+    },
+    {
+      "code": "LT-21",
+      "name": "Klaipėda",
+      "parent": "LT-KL",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-22",
+      "name": "Kretinga",
+      "parent": "LT-KL",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-23",
+      "name": "Kupiškis",
+      "parent": "LT-PN",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-24",
+      "name": "Lazdijai",
+      "parent": "LT-AL",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-25",
+      "name": "Marijampolė",
+      "parent": "LT-MR",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-26",
+      "name": "Mažeikiai",
+      "parent": "LT-TE",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-27",
+      "name": "Molėtai",
+      "parent": "LT-UT",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-28",
+      "name": "Neringa",
+      "parent": "LT-KL",
+      "type": "Municipality"
+    },
+    {
+      "code": "LT-29",
+      "name": "Pagėgiai",
+      "parent": "LT-TA",
+      "type": "Municipality"
+    },
+    {
+      "code": "LT-30",
+      "name": "Pakruojis",
+      "parent": "LT-SA",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-31",
+      "name": "Palangos miestas",
+      "parent": "LT-KL",
+      "type": "City municipality"
+    },
+    {
+      "code": "LT-32",
+      "name": "Panevėžio miestas",
+      "parent": "LT-PN",
+      "type": "City municipality"
+    },
+    {
+      "code": "LT-33",
+      "name": "Panevėžys",
+      "parent": "LT-PN",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-34",
+      "name": "Pasvalys",
+      "parent": "LT-PN",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-35",
+      "name": "Plungė",
+      "parent": "LT-TE",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-36",
+      "name": "Prienai",
+      "parent": "LT-KU",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-37",
+      "name": "Radviliškis",
+      "parent": "LT-SA",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-38",
+      "name": "Raseiniai",
+      "parent": "LT-KU",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-39",
+      "name": "Rietavas",
+      "parent": "LT-TE",
+      "type": "Municipality"
+    },
+    {
+      "code": "LT-40",
+      "name": "Rokiškis",
+      "parent": "LT-PN",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-41",
+      "name": "Šakiai",
+      "parent": "LT-MR",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-42",
+      "name": "Šalčininkai",
+      "parent": "LT-VL",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-43",
+      "name": "Šiaulių miestas",
+      "parent": "LT-SA",
+      "type": "City municipality"
+    },
+    {
+      "code": "LT-44",
+      "name": "Šiauliai",
+      "parent": "LT-SA",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-45",
+      "name": "Šilalė",
+      "parent": "LT-TA",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-46",
+      "name": "Šilutė",
+      "parent": "LT-KL",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-47",
+      "name": "Širvintos",
+      "parent": "LT-VL",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-48",
+      "name": "Skuodas",
+      "parent": "LT-KL",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-49",
+      "name": "Švenčionys",
+      "parent": "LT-VL",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-50",
+      "name": "Tauragė",
+      "parent": "LT-TA",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-51",
+      "name": "Telšiai",
+      "parent": "LT-TE",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-52",
+      "name": "Trakai",
+      "parent": "LT-VL",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-53",
+      "name": "Ukmergė",
+      "parent": "LT-VL",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-54",
+      "name": "Utena",
+      "parent": "LT-UT",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-55",
+      "name": "Varėna",
+      "parent": "LT-AL",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-56",
+      "name": "Vilkaviškis",
+      "parent": "LT-MR",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-57",
+      "name": "Vilniaus miestas",
+      "parent": "LT-VL",
+      "type": "City municipality"
+    },
+    {
+      "code": "LT-58",
+      "name": "Vilnius",
+      "parent": "LT-VL",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-59",
+      "name": "Visaginas",
+      "parent": "LT-UT",
+      "type": "Municipality"
+    },
+    {
+      "code": "LT-60",
+      "name": "Zarasai",
+      "parent": "LT-UT",
+      "type": "District municipality"
+    },
+    {
+      "code": "LT-AL",
+      "name": "Alytaus apskritis",
+      "type": "County"
+    },
+    {
+      "code": "LT-KL",
+      "name": "Klaipėdos apskritis",
+      "type": "County"
+    },
+    {
+      "code": "LT-KU",
+      "name": "Kauno apskritis",
+      "type": "County"
+    },
+    {
+      "code": "LT-MR",
+      "name": "Marijampolės apskritis",
+      "type": "County"
+    },
+    {
+      "code": "LT-PN",
+      "name": "Panevėžio apskritis",
+      "type": "County"
+    },
+    {
+      "code": "LT-SA",
+      "name": "Šiaulių apskritis",
+      "type": "County"
+    },
+    {
+      "code": "LT-TA",
+      "name": "Tauragės apskritis",
+      "type": "County"
+    },
+    {
+      "code": "LT-TE",
+      "name": "Telšių apskritis",
+      "type": "County"
+    },
+    {
+      "code": "LT-UT",
+      "name": "Utenos apskritis",
+      "type": "County"
+    },
+    {
+      "code": "LT-VL",
+      "name": "Vilniaus apskritis",
+      "type": "County"
+    },
+    {
+      "code": "LU-CA",
+      "name": "Capellen",
+      "type": "Canton"
+    },
+    {
+      "code": "LU-CL",
+      "name": "Clervaux",
+      "type": "Canton"
+    },
+    {
+      "code": "LU-DI",
+      "name": "Diekirch",
+      "type": "Canton"
+    },
+    {
+      "code": "LU-EC",
+      "name": "Echternach",
+      "type": "Canton"
+    },
+    {
+      "code": "LU-ES",
+      "name": "Esch-sur-Alzette",
+      "type": "Canton"
+    },
+    {
+      "code": "LU-GR",
+      "name": "Grevenmacher",
+      "type": "Canton"
+    },
+    {
+      "code": "LU-LU",
+      "name": "Luxembourg",
+      "type": "Canton"
+    },
+    {
+      "code": "LU-ME",
+      "name": "Mersch",
+      "type": "Canton"
+    },
+    {
+      "code": "LU-RD",
+      "name": "Redange",
+      "type": "Canton"
+    },
+    {
+      "code": "LU-RM",
+      "name": "Remich",
+      "type": "Canton"
+    },
+    {
+      "code": "LU-VD",
+      "name": "Vianden",
+      "type": "Canton"
+    },
+    {
+      "code": "LU-WI",
+      "name": "Wiltz",
+      "type": "Canton"
+    },
+    {
+      "code": "LV-002",
+      "name": "Aizkraukles novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-007",
+      "name": "Alūksnes novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-011",
+      "name": "Ādažu novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-015",
+      "name": "Balvu novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-016",
+      "name": "Bauskas novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-022",
+      "name": "Cēsu novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-026",
+      "name": "Dobeles novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-033",
+      "name": "Gulbenes novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-041",
+      "name": "Jelgavas novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-042",
+      "name": "Jēkabpils novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-047",
+      "name": "Krāslavas novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-050",
+      "name": "Kuldīgas novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-052",
+      "name": "Ķekavas novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-054",
+      "name": "Limbažu novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-056",
+      "name": "Līvānu novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-058",
+      "name": "Ludzas novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-059",
+      "name": "Madonas novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-062",
+      "name": "Mārupes novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-067",
+      "name": "Ogres novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-068",
+      "name": "Olaines novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-073",
+      "name": "Preiļu novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-077",
+      "name": "Rēzeknes novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-080",
+      "name": "Ropažu novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-087",
+      "name": "Salaspils novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-088",
+      "name": "Saldus novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-089",
+      "name": "Saulkrastu novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-091",
+      "name": "Siguldas novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-094",
+      "name": "Smiltenes novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-097",
+      "name": "Talsu novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-099",
+      "name": "Tukuma novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-101",
+      "name": "Valkas novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-102",
+      "name": "Varakļānu novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-106",
+      "name": "Ventspils novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-111",
+      "name": "Augšdaugavas novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-112",
+      "name": "Dienvidkurzemes Novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-113",
+      "name": "Valmieras Novads",
+      "type": "Municipality"
+    },
+    {
+      "code": "LV-DGV",
+      "name": "Daugavpils",
+      "type": "State city"
+    },
+    {
+      "code": "LV-JEL",
+      "name": "Jelgava",
+      "type": "State city"
+    },
+    {
+      "code": "LV-JUR",
+      "name": "Jūrmala",
+      "type": "State city"
+    },
+    {
+      "code": "LV-LPX",
+      "name": "Liepāja",
+      "type": "State city"
+    },
+    {
+      "code": "LV-REZ",
+      "name": "Rēzekne",
+      "type": "State city"
+    },
+    {
+      "code": "LV-RIX",
+      "name": "Rīga",
+      "type": "State city"
+    },
+    {
+      "code": "LV-VEN",
+      "name": "Ventspils",
+      "type": "State city"
+    },
+    {
+      "code": "LY-BA",
+      "name": "Banghāzī",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-BU",
+      "name": "Al Buţnān",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-DR",
+      "name": "Darnah",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-GT",
+      "name": "Ghāt",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-JA",
+      "name": "Al Jabal al Akhḑar",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-JG",
+      "name": "Al Jabal al Gharbī",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-JI",
+      "name": "Al Jafārah",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-JU",
+      "name": "Al Jufrah",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-KF",
+      "name": "Al Kufrah",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-MB",
+      "name": "Al Marqab",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-MI",
+      "name": "Mişrātah",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-MJ",
+      "name": "Al Marj",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-MQ",
+      "name": "Murzuq",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-NL",
+      "name": "Nālūt",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-NQ",
+      "name": "An Nuqāţ al Khams",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-SB",
+      "name": "Sabhā",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-SR",
+      "name": "Surt",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-TB",
+      "name": "Ţarābulus",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-WA",
+      "name": "Al Wāḩāt",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-WD",
+      "name": "Wādī al Ḩayāt",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-WS",
+      "name": "Wādī ash Shāţi’",
+      "type": "Popularate"
+    },
+    {
+      "code": "LY-ZA",
+      "name": "Az Zāwiyah",
+      "type": "Popularate"
+    },
+    {
+      "code": "MA-01",
+      "name": "Tanger-Tétouan-Al Hoceïma",
+      "type": "Region"
+    },
+    {
+      "code": "MA-02",
+      "name": "L'Oriental",
+      "type": "Region"
+    },
+    {
+      "code": "MA-03",
+      "name": "Fès-Meknès",
+      "type": "Region"
+    },
+    {
+      "code": "MA-04",
+      "name": "Rabat-Salé-Kénitra",
+      "type": "Region"
+    },
+    {
+      "code": "MA-05",
+      "name": "Béni Mellal-Khénifra",
+      "type": "Region"
+    },
+    {
+      "code": "MA-06",
+      "name": "Casablanca-Settat",
+      "type": "Region"
+    },
+    {
+      "code": "MA-07",
+      "name": "Marrakech-Safi",
+      "type": "Region"
+    },
+    {
+      "code": "MA-08",
+      "name": "Drâa-Tafilalet",
+      "type": "Region"
+    },
+    {
+      "code": "MA-09",
+      "name": "Souss-Massa",
+      "type": "Region"
+    },
+    {
+      "code": "MA-10",
+      "name": "Guelmim-Oued Noun (EH-partial)",
+      "type": "Region"
+    },
+    {
+      "code": "MA-11",
+      "name": "Laâyoune-Sakia El Hamra (EH-partial)",
+      "type": "Region"
+    },
+    {
+      "code": "MA-12",
+      "name": "Dakhla-Oued Ed-Dahab (EH)",
+      "type": "Region"
+    },
+    {
+      "code": "MA-AGD",
+      "name": "Agadir-Ida-Ou-Tanane",
+      "parent": "MA-09",
+      "type": "Prefecture"
+    },
+    {
+      "code": "MA-AOU",
+      "name": "Aousserd (EH)",
+      "parent": "MA-12",
+      "type": "Province"
+    },
+    {
+      "code": "MA-ASZ",
+      "name": "Assa-Zag (EH-partial)",
+      "parent": "MA-10",
+      "type": "Province"
+    },
+    {
+      "code": "MA-AZI",
+      "name": "Azilal",
+      "parent": "MA-05",
+      "type": "Province"
+    },
+    {
+      "code": "MA-BEM",
+      "name": "Béni Mellal",
+      "parent": "MA-05",
+      "type": "Province"
+    },
+    {
+      "code": "MA-BER",
+      "name": "Berkane",
+      "parent": "MA-02",
+      "type": "Province"
+    },
+    {
+      "code": "MA-BES",
+      "name": "Benslimane",
+      "parent": "MA-06",
+      "type": "Province"
+    },
+    {
+      "code": "MA-BOD",
+      "name": "Boujdour (EH)",
+      "parent": "MA-11",
+      "type": "Province"
+    },
+    {
+      "code": "MA-BOM",
+      "name": "Boulemane",
+      "parent": "MA-03",
+      "type": "Province"
+    },
+    {
+      "code": "MA-BRR",
+      "name": "Berrechid",
+      "parent": "MA-06",
+      "type": "Province"
+    },
+    {
+      "code": "MA-CAS",
+      "name": "Casablanca",
+      "parent": "MA-06",
+      "type": "Prefecture"
+    },
+    {
+      "code": "MA-CHE",
+      "name": "Chefchaouen",
+      "parent": "MA-01",
+      "type": "Province"
+    },
+    {
+      "code": "MA-CHI",
+      "name": "Chichaoua",
+      "parent": "MA-07",
+      "type": "Province"
+    },
+    {
+      "code": "MA-CHT",
+      "name": "Chtouka-Ait Baha",
+      "parent": "MA-06",
+      "type": "Province"
+    },
+    {
+      "code": "MA-DRI",
+      "name": "Driouch",
+      "parent": "MA-02",
+      "type": "Province"
+    },
+    {
+      "code": "MA-ERR",
+      "name": "Errachidia",
+      "parent": "MA-08",
+      "type": "Province"
+    },
+    {
+      "code": "MA-ESI",
+      "name": "Essaouira",
+      "parent": "MA-07",
+      "type": "Province"
+    },
+    {
+      "code": "MA-ESM",
+      "name": "Es-Semara (EH-partial)",
+      "parent": "MA-11",
+      "type": "Province"
+    },
+    {
+      "code": "MA-FAH",
+      "name": "Fahs-Anjra",
+      "parent": "MA-01",
+      "type": "Province"
+    },
+    {
+      "code": "MA-FES",
+      "name": "Fès",
+      "parent": "MA-03",
+      "type": "Prefecture"
+    },
+    {
+      "code": "MA-FIG",
+      "name": "Figuig",
+      "parent": "MA-02",
+      "type": "Province"
+    },
+    {
+      "code": "MA-FQH",
+      "name": "Fquih Ben Salah",
+      "parent": "MA-05",
+      "type": "Province"
+    },
+    {
+      "code": "MA-GUE",
+      "name": "Guelmim",
+      "parent": "MA-10",
+      "type": "Province"
+    },
+    {
+      "code": "MA-GUF",
+      "name": "Guercif",
+      "parent": "MA-02",
+      "type": "Province"
+    },
+    {
+      "code": "MA-HAJ",
+      "name": "El Hajeb",
+      "parent": "MA-03",
+      "type": "Province"
+    },
+    {
+      "code": "MA-HAO",
+      "name": "Al Haouz",
+      "parent": "MA-07",
+      "type": "Province"
+    },
+    {
+      "code": "MA-HOC",
+      "name": "Al Hoceïma",
+      "parent": "MA-01",
+      "type": "Province"
+    },
+    {
+      "code": "MA-IFR",
+      "name": "Ifrane",
+      "parent": "MA-03",
+      "type": "Province"
+    },
+    {
+      "code": "MA-INE",
+      "name": "Inezgane-Ait Melloul",
+      "parent": "MA-09",
+      "type": "Prefecture"
+    },
+    {
+      "code": "MA-JDI",
+      "name": "El Jadida",
+      "parent": "MA-06",
+      "type": "Province"
+    },
+    {
+      "code": "MA-JRA",
+      "name": "Jerada",
+      "parent": "MA-02",
+      "type": "Province"
+    },
+    {
+      "code": "MA-KEN",
+      "name": "Kénitra",
+      "parent": "MA-04",
+      "type": "Province"
+    },
+    {
+      "code": "MA-KES",
+      "name": "El Kelâa des Sraghna",
+      "parent": "MA-07",
+      "type": "Province"
+    },
+    {
+      "code": "MA-KHE",
+      "name": "Khémisset",
+      "parent": "MA-04",
+      "type": "Province"
+    },
+    {
+      "code": "MA-KHN",
+      "name": "Khénifra",
+      "parent": "MA-05",
+      "type": "Province"
+    },
+    {
+      "code": "MA-KHO",
+      "name": "Khouribga",
+      "parent": "MA-05",
+      "type": "Province"
+    },
+    {
+      "code": "MA-LAA",
+      "name": "Laâyoune (EH)",
+      "parent": "MA-11",
+      "type": "Province"
+    },
+    {
+      "code": "MA-LAR",
+      "name": "Larache",
+      "parent": "MA-01",
+      "type": "Province"
+    },
+    {
+      "code": "MA-MAR",
+      "name": "Marrakech",
+      "parent": "MA-07",
+      "type": "Prefecture"
+    },
+    {
+      "code": "MA-MDF",
+      "name": "M’diq-Fnideq",
+      "parent": "MA-01",
+      "type": "Prefecture"
+    },
+    {
+      "code": "MA-MED",
+      "name": "Médiouna",
+      "parent": "MA-06",
+      "type": "Province"
+    },
+    {
+      "code": "MA-MEK",
+      "name": "Meknès",
+      "parent": "MA-03",
+      "type": "Prefecture"
+    },
+    {
+      "code": "MA-MID",
+      "name": "Midelt",
+      "parent": "MA-08",
+      "type": "Province"
+    },
+    {
+      "code": "MA-MOH",
+      "name": "Mohammadia",
+      "parent": "MA-06",
+      "type": "Prefecture"
+    },
+    {
+      "code": "MA-MOU",
+      "name": "Moulay Yacoub",
+      "parent": "MA-03",
+      "type": "Province"
+    },
+    {
+      "code": "MA-NAD",
+      "name": "Nador",
+      "parent": "MA-02",
+      "type": "Province"
+    },
+    {
+      "code": "MA-NOU",
+      "name": "Nouaceur",
+      "parent": "MA-04",
+      "type": "Province"
+    },
+    {
+      "code": "MA-OUA",
+      "name": "Ouarzazate",
+      "parent": "MA-08",
+      "type": "Province"
+    },
+    {
+      "code": "MA-OUD",
+      "name": "Oued Ed-Dahab (EH)",
+      "parent": "MA-12",
+      "type": "Province"
+    },
+    {
+      "code": "MA-OUJ",
+      "name": "Oujda-Angad",
+      "parent": "MA-02",
+      "type": "Prefecture"
+    },
+    {
+      "code": "MA-OUZ",
+      "name": "Ouezzane",
+      "parent": "MA-01",
+      "type": "Province"
+    },
+    {
+      "code": "MA-RAB",
+      "name": "Rabat",
+      "parent": "MA-04",
+      "type": "Prefecture"
+    },
+    {
+      "code": "MA-REH",
+      "name": "Rehamna",
+      "parent": "MA-07",
+      "type": "Province"
+    },
+    {
+      "code": "MA-SAF",
+      "name": "Safi",
+      "parent": "MA-07",
+      "type": "Province"
+    },
+    {
+      "code": "MA-SAL",
+      "name": "Salé",
+      "parent": "MA-04",
+      "type": "Prefecture"
+    },
+    {
+      "code": "MA-SEF",
+      "name": "Sefrou",
+      "parent": "MA-03",
+      "type": "Province"
+    },
+    {
+      "code": "MA-SET",
+      "name": "Settat",
+      "parent": "MA-06",
+      "type": "Province"
+    },
+    {
+      "code": "MA-SIB",
+      "name": "Sidi Bennour",
+      "parent": "MA-06",
+      "type": "Province"
+    },
+    {
+      "code": "MA-SIF",
+      "name": "Sidi Ifni",
+      "parent": "MA-10",
+      "type": "Province"
+    },
+    {
+      "code": "MA-SIK",
+      "name": "Sidi Kacem",
+      "parent": "MA-04",
+      "type": "Province"
+    },
+    {
+      "code": "MA-SIL",
+      "name": "Sidi Slimane",
+      "parent": "MA-04",
+      "type": "Province"
+    },
+    {
+      "code": "MA-SKH",
+      "name": "Skhirate-Témara",
+      "parent": "MA-04",
+      "type": "Prefecture"
+    },
+    {
+      "code": "MA-TAF",
+      "name": "Tarfaya (EH-partial)",
+      "parent": "MA-11",
+      "type": "Province"
+    },
+    {
+      "code": "MA-TAI",
+      "name": "Taourirt",
+      "parent": "MA-02",
+      "type": "Province"
+    },
+    {
+      "code": "MA-TAO",
+      "name": "Taounate",
+      "parent": "MA-03",
+      "type": "Province"
+    },
+    {
+      "code": "MA-TAR",
+      "name": "Taroudannt",
+      "parent": "MA-09",
+      "type": "Province"
+    },
+    {
+      "code": "MA-TAT",
+      "name": "Tata",
+      "parent": "MA-09",
+      "type": "Province"
+    },
+    {
+      "code": "MA-TAZ",
+      "name": "Taza",
+      "parent": "MA-03",
+      "type": "Province"
+    },
+    {
+      "code": "MA-TET",
+      "name": "Tétouan",
+      "parent": "MA-01",
+      "type": "Province"
+    },
+    {
+      "code": "MA-TIN",
+      "name": "Tinghir",
+      "parent": "MA-08",
+      "type": "Province"
+    },
+    {
+      "code": "MA-TIZ",
+      "name": "Tiznit",
+      "parent": "MA-09",
+      "type": "Province"
+    },
+    {
+      "code": "MA-TNG",
+      "name": "Tanger-Assilah",
+      "parent": "MA-01",
+      "type": "Prefecture"
+    },
+    {
+      "code": "MA-TNT",
+      "name": "Tan-Tan (EH-partial)",
+      "parent": "MA-10",
+      "type": "Province"
+    },
+    {
+      "code": "MA-YUS",
+      "name": "Youssoufia",
+      "parent": "MA-07",
+      "type": "Province"
+    },
+    {
+      "code": "MA-ZAG",
+      "name": "Zagora",
+      "parent": "MA-08",
+      "type": "Province"
+    },
+    {
+      "code": "MC-CL",
+      "name": "La Colle",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-CO",
+      "name": "La Condamine",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-FO",
+      "name": "Fontvieille",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-GA",
+      "name": "La Gare",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-JE",
+      "name": "Jardin Exotique",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-LA",
+      "name": "Larvotto",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-MA",
+      "name": "Malbousquet",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-MC",
+      "name": "Monte-Carlo",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-MG",
+      "name": "Moneghetti",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-MO",
+      "name": "Monaco-Ville",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-MU",
+      "name": "Moulins",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-PH",
+      "name": "Port-Hercule",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-SD",
+      "name": "Sainte-Dévote",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-SO",
+      "name": "La Source",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-SP",
+      "name": "Spélugues",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-SR",
+      "name": "Saint-Roman",
+      "type": "Quarter"
+    },
+    {
+      "code": "MC-VR",
+      "name": "Vallon de la Rousse",
+      "type": "Quarter"
+    },
+    {
+      "code": "MD-AN",
+      "name": "Anenii Noi",
+      "type": "District"
+    },
+    {
+      "code": "MD-BA",
+      "name": "Bălți",
+      "type": "City"
+    },
+    {
+      "code": "MD-BD",
+      "name": "Bender [Tighina]",
+      "type": "City"
+    },
+    {
+      "code": "MD-BR",
+      "name": "Briceni",
+      "type": "District"
+    },
+    {
+      "code": "MD-BS",
+      "name": "Basarabeasca",
+      "type": "District"
+    },
+    {
+      "code": "MD-CA",
+      "name": "Cahul",
+      "type": "District"
+    },
+    {
+      "code": "MD-CL",
+      "name": "Călărași",
+      "type": "District"
+    },
+    {
+      "code": "MD-CM",
+      "name": "Cimișlia",
+      "type": "District"
+    },
+    {
+      "code": "MD-CR",
+      "name": "Criuleni",
+      "type": "District"
+    },
+    {
+      "code": "MD-CS",
+      "name": "Căușeni",
+      "type": "District"
+    },
+    {
+      "code": "MD-CT",
+      "name": "Cantemir",
+      "type": "District"
+    },
+    {
+      "code": "MD-CU",
+      "name": "Chișinău",
+      "type": "City"
+    },
+    {
+      "code": "MD-DO",
+      "name": "Dondușeni",
+      "type": "District"
+    },
+    {
+      "code": "MD-DR",
+      "name": "Drochia",
+      "type": "District"
+    },
+    {
+      "code": "MD-DU",
+      "name": "Dubăsari",
+      "type": "District"
+    },
+    {
+      "code": "MD-ED",
+      "name": "Edineț",
+      "type": "District"
+    },
+    {
+      "code": "MD-FA",
+      "name": "Fălești",
+      "type": "District"
+    },
+    {
+      "code": "MD-FL",
+      "name": "Florești",
+      "type": "District"
+    },
+    {
+      "code": "MD-GA",
+      "name": "Găgăuzia, Unitatea teritorială autonomă (UTAG)",
+      "type": "Autonomous territorial unit"
+    },
+    {
+      "code": "MD-GL",
+      "name": "Glodeni",
+      "type": "District"
+    },
+    {
+      "code": "MD-HI",
+      "name": "Hîncești",
+      "type": "District"
+    },
+    {
+      "code": "MD-IA",
+      "name": "Ialoveni",
+      "type": "District"
+    },
+    {
+      "code": "MD-LE",
+      "name": "Leova",
+      "type": "District"
+    },
+    {
+      "code": "MD-NI",
+      "name": "Nisporeni",
+      "type": "District"
+    },
+    {
+      "code": "MD-OC",
+      "name": "Ocnița",
+      "type": "District"
+    },
+    {
+      "code": "MD-OR",
+      "name": "Orhei",
+      "type": "District"
+    },
+    {
+      "code": "MD-RE",
+      "name": "Rezina",
+      "type": "District"
+    },
+    {
+      "code": "MD-RI",
+      "name": "Rîșcani",
+      "type": "District"
+    },
+    {
+      "code": "MD-SD",
+      "name": "Șoldănești",
+      "type": "District"
+    },
+    {
+      "code": "MD-SI",
+      "name": "Sîngerei",
+      "type": "District"
+    },
+    {
+      "code": "MD-SN",
+      "name": "Stînga Nistrului, unitatea teritorială din",
+      "type": "Territorial unit"
+    },
+    {
+      "code": "MD-SO",
+      "name": "Soroca",
+      "type": "District"
+    },
+    {
+      "code": "MD-ST",
+      "name": "Strășeni",
+      "type": "District"
+    },
+    {
+      "code": "MD-SV",
+      "name": "Ștefan Vodă",
+      "type": "District"
+    },
+    {
+      "code": "MD-TA",
+      "name": "Taraclia",
+      "type": "District"
+    },
+    {
+      "code": "MD-TE",
+      "name": "Telenești",
+      "type": "District"
+    },
+    {
+      "code": "MD-UN",
+      "name": "Ungheni",
+      "type": "District"
+    },
+    {
+      "code": "ME-01",
+      "name": "Andrijevica",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-02",
+      "name": "Bar",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-03",
+      "name": "Berane",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-04",
+      "name": "Bijelo Polje",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-05",
+      "name": "Budva",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-06",
+      "name": "Cetinje",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-07",
+      "name": "Danilovgrad",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-08",
+      "name": "Herceg-Novi",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-09",
+      "name": "Kolašin",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-10",
+      "name": "Kotor",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-11",
+      "name": "Mojkovac",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-12",
+      "name": "Nikšić",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-13",
+      "name": "Plav",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-14",
+      "name": "Pljevlja",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-15",
+      "name": "Plužine",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-16",
+      "name": "Podgorica",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-17",
+      "name": "Rožaje",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-18",
+      "name": "Šavnik",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-19",
+      "name": "Tivat",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-20",
+      "name": "Ulcinj",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-21",
+      "name": "Žabljak",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-22",
+      "name": "Gusinje",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-23",
+      "name": "Petnjica",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-24",
+      "name": "Tuzi",
+      "type": "Municipality"
+    },
+    {
+      "code": "ME-25",
+      "name": "Zeta",
+      "type": "Municipality"
+    },
+    {
+      "code": "MG-A",
+      "name": "Toamasina",
+      "type": "Province"
+    },
+    {
+      "code": "MG-D",
+      "name": "Antsiranana",
+      "type": "Province"
+    },
+    {
+      "code": "MG-F",
+      "name": "Fianarantsoa",
+      "type": "Province"
+    },
+    {
+      "code": "MG-M",
+      "name": "Mahajanga",
+      "type": "Province"
+    },
+    {
+      "code": "MG-T",
+      "name": "Antananarivo",
+      "type": "Province"
+    },
+    {
+      "code": "MG-U",
+      "name": "Toliara",
+      "type": "Province"
+    },
+    {
+      "code": "MH-ALK",
+      "name": "Ailuk",
+      "parent": "MH-T",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-ALL",
+      "name": "Ailinglaplap",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-ARN",
+      "name": "Arno",
+      "parent": "MH-T",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-AUR",
+      "name": "Aur",
+      "parent": "MH-T",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-EBO",
+      "name": "Ebon",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-ENI",
+      "name": "Enewetak & Ujelang",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-JAB",
+      "name": "Jabat",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-JAL",
+      "name": "Jaluit",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-KIL",
+      "name": "Bikini & Kili",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-KWA",
+      "name": "Kwajalein",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-L",
+      "name": "Ralik chain",
+      "type": "Chain (of islands)"
+    },
+    {
+      "code": "MH-LAE",
+      "name": "Lae",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-LIB",
+      "name": "Lib",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-LIK",
+      "name": "Likiep",
+      "parent": "MH-T",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-MAJ",
+      "name": "Majuro",
+      "parent": "MH-T",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-MAL",
+      "name": "Maloelap",
+      "parent": "MH-T",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-MEJ",
+      "name": "Mejit",
+      "parent": "MH-T",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-MIL",
+      "name": "Mili",
+      "parent": "MH-T",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-NMK",
+      "name": "Namdrik",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-NMU",
+      "name": "Namu",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-RON",
+      "name": "Rongelap",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-T",
+      "name": "Ratak chain",
+      "type": "Chain (of islands)"
+    },
+    {
+      "code": "MH-UJA",
+      "name": "Ujae",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-UTI",
+      "name": "Utrik",
+      "parent": "MH-T",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-WTH",
+      "name": "Wotho",
+      "parent": "MH-L",
+      "type": "Municipality"
+    },
+    {
+      "code": "MH-WTJ",
+      "name": "Wotje",
+      "parent": "MH-T",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-101",
+      "name": "Veles",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-102",
+      "name": "Gradsko",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-103",
+      "name": "Demir Kapija",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-104",
+      "name": "Kavadarci",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-105",
+      "name": "Lozovo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-106",
+      "name": "Negotino",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-107",
+      "name": "Rosoman",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-108",
+      "name": "Sveti Nikole",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-109",
+      "name": "Čaška",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-201",
+      "name": "Berovo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-202",
+      "name": "Vinica",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-203",
+      "name": "Delčevo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-204",
+      "name": "Zrnovci",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-205",
+      "name": "Karbinci",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-206",
+      "name": "Kočani",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-207",
+      "name": "Makedonska Kamenica",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-208",
+      "name": "Pehčevo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-209",
+      "name": "Probištip",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-210",
+      "name": "Češinovo-Obleševo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-211",
+      "name": "Štip",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-301",
+      "name": "Vevčani",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-303",
+      "name": "Debar",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-304",
+      "name": "Debrca",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-307",
+      "name": "Kičevo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-308",
+      "name": "Makedonski Brod",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-310",
+      "name": "Ohrid",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-311",
+      "name": "Plasnica",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-312",
+      "name": "Struga",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-313",
+      "name": "Centar Župa",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-401",
+      "name": "Bogdanci",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-402",
+      "name": "Bosilovo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-403",
+      "name": "Valandovo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-404",
+      "name": "Vasilevo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-405",
+      "name": "Gevgelija",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-406",
+      "name": "Dojran",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-407",
+      "name": "Konče",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-408",
+      "name": "Novo Selo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-409",
+      "name": "Radoviš",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-410",
+      "name": "Strumica",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-501",
+      "name": "Bitola",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-502",
+      "name": "Demir Hisar",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-503",
+      "name": "Dolneni",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-504",
+      "name": "Krivogaštani",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-505",
+      "name": "Kruševo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-506",
+      "name": "Mogila",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-507",
+      "name": "Novaci",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-508",
+      "name": "Prilep",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-509",
+      "name": "Resen",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-601",
+      "name": "Bogovinje",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-602",
+      "name": "Brvenica",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-603",
+      "name": "Vrapčište",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-604",
+      "name": "Gostivar",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-605",
+      "name": "Želino",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-606",
+      "name": "Jegunovce",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-607",
+      "name": "Mavrovo i Rostuše",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-608",
+      "name": "Tearce",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-609",
+      "name": "Tetovo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-701",
+      "name": "Kratovo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-702",
+      "name": "Kriva Palanka",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-703",
+      "name": "Kumanovo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-704",
+      "name": "Lipkovo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-705",
+      "name": "Rankovce",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-706",
+      "name": "Staro Nagoričane",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-801",
+      "name": "Aerodrom †",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-802",
+      "name": "Aračinovo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-803",
+      "name": "Butel †",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-804",
+      "name": "Gazi Baba †",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-805",
+      "name": "Gjorče Petrov †",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-806",
+      "name": "Zelenikovo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-807",
+      "name": "Ilinden",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-808",
+      "name": "Karpoš †",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-809",
+      "name": "Kisela Voda †",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-810",
+      "name": "Petrovec",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-811",
+      "name": "Saraj †",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-812",
+      "name": "Sopište",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-813",
+      "name": "Studeničani",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-814",
+      "name": "Centar †",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-815",
+      "name": "Čair †",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-816",
+      "name": "Čučer-Sandevo",
+      "type": "Municipality"
+    },
+    {
+      "code": "MK-817",
+      "name": "Šuto Orizari †",
+      "type": "Municipality"
+    },
+    {
+      "code": "ML-1",
+      "name": "Kayes",
+      "type": "Region"
+    },
+    {
+      "code": "ML-10",
+      "name": "Taoudénit",
+      "type": "Region"
+    },
+    {
+      "code": "ML-2",
+      "name": "Koulikoro",
+      "type": "Region"
+    },
+    {
+      "code": "ML-3",
+      "name": "Sikasso",
+      "type": "Region"
+    },
+    {
+      "code": "ML-4",
+      "name": "Ségou",
+      "type": "Region"
+    },
+    {
+      "code": "ML-5",
+      "name": "Mopti",
+      "type": "Region"
+    },
+    {
+      "code": "ML-6",
+      "name": "Tombouctou",
+      "type": "Region"
+    },
+    {
+      "code": "ML-7",
+      "name": "Gao",
+      "type": "Region"
+    },
+    {
+      "code": "ML-8",
+      "name": "Kidal",
+      "type": "Region"
+    },
+    {
+      "code": "ML-9",
+      "name": "Ménaka",
+      "type": "Region"
+    },
+    {
+      "code": "ML-BKO",
+      "name": "Bamako",
+      "type": "District"
+    },
+    {
+      "code": "MM-01",
+      "name": "Sagaing",
+      "type": "Region"
+    },
+    {
+      "code": "MM-02",
+      "name": "Bago",
+      "type": "Region"
+    },
+    {
+      "code": "MM-03",
+      "name": "Magway",
+      "type": "Region"
+    },
+    {
+      "code": "MM-04",
+      "name": "Mandalay",
+      "type": "Region"
+    },
+    {
+      "code": "MM-05",
+      "name": "Tanintharyi",
+      "type": "Region"
+    },
+    {
+      "code": "MM-06",
+      "name": "Yangon",
+      "type": "Region"
+    },
+    {
+      "code": "MM-07",
+      "name": "Ayeyarwady",
+      "type": "Region"
+    },
+    {
+      "code": "MM-11",
+      "name": "Kachin",
+      "type": "State"
+    },
+    {
+      "code": "MM-12",
+      "name": "Kayah",
+      "type": "State"
+    },
+    {
+      "code": "MM-13",
+      "name": "Kayin",
+      "type": "State"
+    },
+    {
+      "code": "MM-14",
+      "name": "Chin",
+      "type": "State"
+    },
+    {
+      "code": "MM-15",
+      "name": "Mon",
+      "type": "State"
+    },
+    {
+      "code": "MM-16",
+      "name": "Rakhine",
+      "type": "State"
+    },
+    {
+      "code": "MM-17",
+      "name": "Shan",
+      "type": "State"
+    },
+    {
+      "code": "MM-18",
+      "name": "Nay Pyi Taw",
+      "type": "Union territory"
+    },
+    {
+      "code": "MN-035",
+      "name": "Orhon",
+      "type": "Province"
+    },
+    {
+      "code": "MN-037",
+      "name": "Darhan uul",
+      "type": "Province"
+    },
+    {
+      "code": "MN-039",
+      "name": "Hentiy",
+      "type": "Province"
+    },
+    {
+      "code": "MN-041",
+      "name": "Hövsgöl",
+      "type": "Province"
+    },
+    {
+      "code": "MN-043",
+      "name": "Hovd",
+      "type": "Province"
+    },
+    {
+      "code": "MN-046",
+      "name": "Uvs",
+      "type": "Province"
+    },
+    {
+      "code": "MN-047",
+      "name": "Töv",
+      "type": "Province"
+    },
+    {
+      "code": "MN-049",
+      "name": "Selenge",
+      "type": "Province"
+    },
+    {
+      "code": "MN-051",
+      "name": "Sühbaatar",
+      "type": "Province"
+    },
+    {
+      "code": "MN-053",
+      "name": "Ömnögovĭ",
+      "type": "Province"
+    },
+    {
+      "code": "MN-055",
+      "name": "Övörhangay",
+      "type": "Province"
+    },
+    {
+      "code": "MN-057",
+      "name": "Dzavhan",
+      "type": "Province"
+    },
+    {
+      "code": "MN-059",
+      "name": "Dundgovĭ",
+      "type": "Province"
+    },
+    {
+      "code": "MN-061",
+      "name": "Dornod",
+      "type": "Province"
+    },
+    {
+      "code": "MN-063",
+      "name": "Dornogovĭ",
+      "type": "Province"
+    },
+    {
+      "code": "MN-064",
+      "name": "Govĭ-Sümber",
+      "type": "Province"
+    },
+    {
+      "code": "MN-065",
+      "name": "Govĭ-Altay",
+      "type": "Province"
+    },
+    {
+      "code": "MN-067",
+      "name": "Bulgan",
+      "type": "Province"
+    },
+    {
+      "code": "MN-069",
+      "name": "Bayanhongor",
+      "type": "Province"
+    },
+    {
+      "code": "MN-071",
+      "name": "Bayan-Ölgiy",
+      "type": "Province"
+    },
+    {
+      "code": "MN-073",
+      "name": "Arhangay",
+      "type": "Province"
+    },
+    {
+      "code": "MN-1",
+      "name": "Ulaanbaatar",
+      "type": "Capital city"
+    },
+    {
+      "code": "MR-01",
+      "name": "Hodh ech Chargui",
+      "type": "Region"
+    },
+    {
+      "code": "MR-02",
+      "name": "Hodh el Gharbi",
+      "type": "Region"
+    },
+    {
+      "code": "MR-03",
+      "name": "Assaba",
+      "type": "Region"
+    },
+    {
+      "code": "MR-04",
+      "name": "Gorgol",
+      "type": "Region"
+    },
+    {
+      "code": "MR-05",
+      "name": "Brakna",
+      "type": "Region"
+    },
+    {
+      "code": "MR-06",
+      "name": "Trarza",
+      "type": "Region"
+    },
+    {
+      "code": "MR-07",
+      "name": "Adrar",
+      "type": "Region"
+    },
+    {
+      "code": "MR-08",
+      "name": "Dakhlet Nouâdhibou",
+      "type": "Region"
+    },
+    {
+      "code": "MR-09",
+      "name": "Tagant",
+      "type": "Region"
+    },
+    {
+      "code": "MR-10",
+      "name": "Guidimaka",
+      "type": "Region"
+    },
+    {
+      "code": "MR-11",
+      "name": "Tiris Zemmour",
+      "type": "Region"
+    },
+    {
+      "code": "MR-12",
+      "name": "Inchiri",
+      "type": "Region"
+    },
+    {
+      "code": "MR-13",
+      "name": "Nouakchott Ouest",
+      "type": "Region"
+    },
+    {
+      "code": "MR-14",
+      "name": "Nouakchott Nord",
+      "type": "Region"
+    },
+    {
+      "code": "MR-15",
+      "name": "Nouakchott Sud",
+      "type": "Region"
+    },
+    {
+      "code": "MT-01",
+      "name": "Attard",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-02",
+      "name": "Balzan",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-03",
+      "name": "Birgu",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-04",
+      "name": "Birkirkara",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-05",
+      "name": "Birżebbuġa",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-06",
+      "name": "Bormla",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-07",
+      "name": "Dingli",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-08",
+      "name": "Fgura",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-09",
+      "name": "Floriana",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-10",
+      "name": "Fontana",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-11",
+      "name": "Gudja",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-12",
+      "name": "Gżira",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-13",
+      "name": "Għajnsielem",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-14",
+      "name": "Għarb",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-15",
+      "name": "Għargħur",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-16",
+      "name": "Għasri",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-17",
+      "name": "Għaxaq",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-18",
+      "name": "Ħamrun",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-19",
+      "name": "Iklin",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-20",
+      "name": "Isla",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-21",
+      "name": "Kalkara",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-22",
+      "name": "Kerċem",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-23",
+      "name": "Kirkop",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-24",
+      "name": "Lija",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-25",
+      "name": "Luqa",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-26",
+      "name": "Marsa",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-27",
+      "name": "Marsaskala",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-28",
+      "name": "Marsaxlokk",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-29",
+      "name": "Mdina",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-30",
+      "name": "Mellieħa",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-31",
+      "name": "Mġarr",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-32",
+      "name": "Mosta",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-33",
+      "name": "Mqabba",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-34",
+      "name": "Msida",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-35",
+      "name": "Mtarfa",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-36",
+      "name": "Munxar",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-37",
+      "name": "Nadur",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-38",
+      "name": "Naxxar",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-39",
+      "name": "Paola",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-40",
+      "name": "Pembroke",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-41",
+      "name": "Pietà",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-42",
+      "name": "Qala",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-43",
+      "name": "Qormi",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-44",
+      "name": "Qrendi",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-45",
+      "name": "Rabat Gozo",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-46",
+      "name": "Rabat Malta",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-47",
+      "name": "Safi",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-48",
+      "name": "Saint Julian's",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-49",
+      "name": "Saint John",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-50",
+      "name": "Saint Lawrence",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-51",
+      "name": "Saint Paul's Bay",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-52",
+      "name": "Sannat",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-53",
+      "name": "Saint Lucia's",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-54",
+      "name": "Santa Venera",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-55",
+      "name": "Siġġiewi",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-56",
+      "name": "Sliema",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-57",
+      "name": "Swieqi",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-58",
+      "name": "Ta' Xbiex",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-59",
+      "name": "Tarxien",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-60",
+      "name": "Valletta",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-61",
+      "name": "Xagħra",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-62",
+      "name": "Xewkija",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-63",
+      "name": "Xgħajra",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-64",
+      "name": "Żabbar",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-65",
+      "name": "Żebbuġ Gozo",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-66",
+      "name": "Żebbuġ Malta",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-67",
+      "name": "Żejtun",
+      "type": "Local council"
+    },
+    {
+      "code": "MT-68",
+      "name": "Żurrieq",
+      "type": "Local council"
+    },
+    {
+      "code": "MU-AG",
+      "name": "Agalega Islands",
+      "type": "Dependency"
+    },
+    {
+      "code": "MU-BL",
+      "name": "Black River",
+      "type": "District"
+    },
+    {
+      "code": "MU-CC",
+      "name": "Cargados Carajos Shoals",
+      "type": "Dependency"
+    },
+    {
+      "code": "MU-FL",
+      "name": "Flacq",
+      "type": "District"
+    },
+    {
+      "code": "MU-GP",
+      "name": "Grand Port",
+      "type": "District"
+    },
+    {
+      "code": "MU-MO",
+      "name": "Moka",
+      "type": "District"
+    },
+    {
+      "code": "MU-PA",
+      "name": "Pamplemousses",
+      "type": "District"
+    },
+    {
+      "code": "MU-PL",
+      "name": "Port Louis",
+      "type": "District"
+    },
+    {
+      "code": "MU-PW",
+      "name": "Plaines Wilhems",
+      "type": "District"
+    },
+    {
+      "code": "MU-RO",
+      "name": "Rodrigues Island",
+      "type": "Dependency"
+    },
+    {
+      "code": "MU-RR",
+      "name": "Rivière du Rempart",
+      "type": "District"
+    },
+    {
+      "code": "MU-SA",
+      "name": "Savanne",
+      "type": "District"
+    },
+    {
+      "code": "MV-00",
+      "name": "South Ari Atoll",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-01",
+      "name": "Addu City",
+      "type": "City"
+    },
+    {
+      "code": "MV-02",
+      "name": "North Ari Atoll",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-03",
+      "name": "Faadhippolhu",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-04",
+      "name": "Felidhu Atoll",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-05",
+      "name": "Hahdhunmathi",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-07",
+      "name": "North Thiladhunmathi",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-08",
+      "name": "Kolhumadulu",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-12",
+      "name": "Mulaku Atoll",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-13",
+      "name": "North Maalhosmadulu",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-14",
+      "name": "North Nilandhe Atoll",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-17",
+      "name": "South Nilandhe Atoll",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-20",
+      "name": "South Maalhosmadulu",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-23",
+      "name": "South Thiladhunmathi",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-24",
+      "name": "North Miladhunmadulu",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-25",
+      "name": "South Miladhunmadulu",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-26",
+      "name": "Male Atoll",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-27",
+      "name": "North Huvadhu Atoll",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-28",
+      "name": "South Huvadhu Atoll",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-29",
+      "name": "Fuvammulah",
+      "type": "Administrative atoll"
+    },
+    {
+      "code": "MV-MLE",
+      "name": "Male",
+      "type": "City"
+    },
+    {
+      "code": "MW-BA",
+      "name": "Balaka",
+      "parent": "MW-S",
+      "type": "District"
+    },
+    {
+      "code": "MW-BL",
+      "name": "Blantyre",
+      "parent": "MW-S",
+      "type": "District"
+    },
+    {
+      "code": "MW-C",
+      "name": "Central Region",
+      "type": "Region"
+    },
+    {
+      "code": "MW-CK",
+      "name": "Chikwawa",
+      "parent": "MW-S",
+      "type": "District"
+    },
+    {
+      "code": "MW-CR",
+      "name": "Chiradzulu",
+      "parent": "MW-S",
+      "type": "District"
+    },
+    {
+      "code": "MW-CT",
+      "name": "Chitipa",
+      "parent": "MW-N",
+      "type": "District"
+    },
+    {
+      "code": "MW-DE",
+      "name": "Dedza",
+      "parent": "MW-C",
+      "type": "District"
+    },
+    {
+      "code": "MW-DO",
+      "name": "Dowa",
+      "parent": "MW-C",
+      "type": "District"
+    },
+    {
+      "code": "MW-KR",
+      "name": "Karonga",
+      "parent": "MW-N",
+      "type": "District"
+    },
+    {
+      "code": "MW-KS",
+      "name": "Kasungu",
+      "parent": "MW-C",
+      "type": "District"
+    },
+    {
+      "code": "MW-LI",
+      "name": "Lilongwe",
+      "parent": "MW-C",
+      "type": "District"
+    },
+    {
+      "code": "MW-LK",
+      "name": "Likoma",
+      "parent": "MW-N",
+      "type": "District"
+    },
+    {
+      "code": "MW-MC",
+      "name": "Mchinji",
+      "parent": "MW-C",
+      "type": "District"
+    },
+    {
+      "code": "MW-MG",
+      "name": "Mangochi",
+      "parent": "MW-S",
+      "type": "District"
+    },
+    {
+      "code": "MW-MH",
+      "name": "Machinga",
+      "parent": "MW-S",
+      "type": "District"
+    },
+    {
+      "code": "MW-MU",
+      "name": "Mulanje",
+      "parent": "MW-S",
+      "type": "District"
+    },
+    {
+      "code": "MW-MW",
+      "name": "Mwanza",
+      "parent": "MW-S",
+      "type": "District"
+    },
+    {
+      "code": "MW-MZ",
+      "name": "Mzimba",
+      "parent": "MW-N",
+      "type": "District"
+    },
+    {
+      "code": "MW-N",
+      "name": "Northern Region",
+      "type": "Region"
+    },
+    {
+      "code": "MW-NB",
+      "name": "Nkhata Bay",
+      "parent": "MW-N",
+      "type": "District"
+    },
+    {
+      "code": "MW-NE",
+      "name": "Neno",
+      "parent": "MW-S",
+      "type": "District"
+    },
+    {
+      "code": "MW-NI",
+      "name": "Ntchisi",
+      "parent": "MW-C",
+      "type": "District"
+    },
+    {
+      "code": "MW-NK",
+      "name": "Nkhotakota",
+      "parent": "MW-C",
+      "type": "District"
+    },
+    {
+      "code": "MW-NS",
+      "name": "Nsanje",
+      "parent": "MW-S",
+      "type": "District"
+    },
+    {
+      "code": "MW-NU",
+      "name": "Ntcheu",
+      "parent": "MW-C",
+      "type": "District"
+    },
+    {
+      "code": "MW-PH",
+      "name": "Phalombe",
+      "parent": "MW-S",
+      "type": "District"
+    },
+    {
+      "code": "MW-RU",
+      "name": "Rumphi",
+      "parent": "MW-N",
+      "type": "District"
+    },
+    {
+      "code": "MW-S",
+      "name": "Southern Region",
+      "type": "Region"
+    },
+    {
+      "code": "MW-SA",
+      "name": "Salima",
+      "parent": "MW-C",
+      "type": "District"
+    },
+    {
+      "code": "MW-TH",
+      "name": "Thyolo",
+      "parent": "MW-S",
+      "type": "District"
+    },
+    {
+      "code": "MW-ZO",
+      "name": "Zomba",
+      "parent": "MW-S",
+      "type": "District"
+    },
+    {
+      "code": "MX-AGU",
+      "name": "Aguascalientes",
+      "type": "State"
+    },
+    {
+      "code": "MX-BCN",
+      "name": "Baja California",
+      "type": "State"
+    },
+    {
+      "code": "MX-BCS",
+      "name": "Baja California Sur",
+      "type": "State"
+    },
+    {
+      "code": "MX-CAM",
+      "name": "Campeche",
+      "type": "State"
+    },
+    {
+      "code": "MX-CHH",
+      "name": "Chihuahua",
+      "type": "State"
+    },
+    {
+      "code": "MX-CHP",
+      "name": "Chiapas",
+      "type": "State"
+    },
+    {
+      "code": "MX-CMX",
+      "name": "Ciudad de México",
+      "type": "Federal entity"
+    },
+    {
+      "code": "MX-COA",
+      "name": "Coahuila de Zaragoza",
+      "type": "State"
+    },
+    {
+      "code": "MX-COL",
+      "name": "Colima",
+      "type": "State"
+    },
+    {
+      "code": "MX-DUR",
+      "name": "Durango",
+      "type": "State"
+    },
+    {
+      "code": "MX-GRO",
+      "name": "Guerrero",
+      "type": "State"
+    },
+    {
+      "code": "MX-GUA",
+      "name": "Guanajuato",
+      "type": "State"
+    },
+    {
+      "code": "MX-HID",
+      "name": "Hidalgo",
+      "type": "State"
+    },
+    {
+      "code": "MX-JAL",
+      "name": "Jalisco",
+      "type": "State"
+    },
+    {
+      "code": "MX-MEX",
+      "name": "México",
+      "type": "State"
+    },
+    {
+      "code": "MX-MIC",
+      "name": "Michoacán de Ocampo",
+      "type": "State"
+    },
+    {
+      "code": "MX-MOR",
+      "name": "Morelos",
+      "type": "State"
+    },
+    {
+      "code": "MX-NAY",
+      "name": "Nayarit",
+      "type": "State"
+    },
+    {
+      "code": "MX-NLE",
+      "name": "Nuevo León",
+      "type": "State"
+    },
+    {
+      "code": "MX-OAX",
+      "name": "Oaxaca",
+      "type": "State"
+    },
+    {
+      "code": "MX-PUE",
+      "name": "Puebla",
+      "type": "State"
+    },
+    {
+      "code": "MX-QUE",
+      "name": "Querétaro",
+      "type": "State"
+    },
+    {
+      "code": "MX-ROO",
+      "name": "Quintana Roo",
+      "type": "State"
+    },
+    {
+      "code": "MX-SIN",
+      "name": "Sinaloa",
+      "type": "State"
+    },
+    {
+      "code": "MX-SLP",
+      "name": "San Luis Potosí",
+      "type": "State"
+    },
+    {
+      "code": "MX-SON",
+      "name": "Sonora",
+      "type": "State"
+    },
+    {
+      "code": "MX-TAB",
+      "name": "Tabasco",
+      "type": "State"
+    },
+    {
+      "code": "MX-TAM",
+      "name": "Tamaulipas",
+      "type": "State"
+    },
+    {
+      "code": "MX-TLA",
+      "name": "Tlaxcala",
+      "type": "State"
+    },
+    {
+      "code": "MX-VER",
+      "name": "Veracruz de Ignacio de la Llave",
+      "type": "State"
+    },
+    {
+      "code": "MX-YUC",
+      "name": "Yucatán",
+      "type": "State"
+    },
+    {
+      "code": "MX-ZAC",
+      "name": "Zacatecas",
+      "type": "State"
+    },
+    {
+      "code": "MY-01",
+      "name": "Johor",
+      "type": "State"
+    },
+    {
+      "code": "MY-02",
+      "name": "Kedah",
+      "type": "State"
+    },
+    {
+      "code": "MY-03",
+      "name": "Kelantan",
+      "type": "State"
+    },
+    {
+      "code": "MY-04",
+      "name": "Melaka",
+      "type": "State"
+    },
+    {
+      "code": "MY-05",
+      "name": "Negeri Sembilan",
+      "type": "State"
+    },
+    {
+      "code": "MY-06",
+      "name": "Pahang",
+      "type": "State"
+    },
+    {
+      "code": "MY-07",
+      "name": "Pulau Pinang",
+      "type": "State"
+    },
+    {
+      "code": "MY-08",
+      "name": "Perak",
+      "type": "State"
+    },
+    {
+      "code": "MY-09",
+      "name": "Perlis",
+      "type": "State"
+    },
+    {
+      "code": "MY-10",
+      "name": "Selangor",
+      "type": "State"
+    },
+    {
+      "code": "MY-11",
+      "name": "Terengganu",
+      "type": "State"
+    },
+    {
+      "code": "MY-12",
+      "name": "Sabah",
+      "type": "State"
+    },
+    {
+      "code": "MY-13",
+      "name": "Sarawak",
+      "type": "State"
+    },
+    {
+      "code": "MY-14",
+      "name": "Wilayah Persekutuan Kuala Lumpur",
+      "type": "Federal territory"
+    },
+    {
+      "code": "MY-15",
+      "name": "Wilayah Persekutuan Labuan",
+      "type": "Federal territory"
+    },
+    {
+      "code": "MY-16",
+      "name": "Wilayah Persekutuan Putrajaya",
+      "type": "Federal territory"
+    },
+    {
+      "code": "MZ-A",
+      "name": "Niassa",
+      "type": "Province"
+    },
+    {
+      "code": "MZ-B",
+      "name": "Manica",
+      "type": "Province"
+    },
+    {
+      "code": "MZ-G",
+      "name": "Gaza",
+      "type": "Province"
+    },
+    {
+      "code": "MZ-I",
+      "name": "Inhambane",
+      "type": "Province"
+    },
+    {
+      "code": "MZ-L",
+      "name": "Maputo",
+      "type": "Province"
+    },
+    {
+      "code": "MZ-MPM",
+      "name": "Maputo",
+      "type": "City"
+    },
+    {
+      "code": "MZ-N",
+      "name": "Nampula",
+      "type": "Province"
+    },
+    {
+      "code": "MZ-P",
+      "name": "Cabo Delgado",
+      "type": "Province"
+    },
+    {
+      "code": "MZ-Q",
+      "name": "Zambézia",
+      "type": "Province"
+    },
+    {
+      "code": "MZ-S",
+      "name": "Sofala",
+      "type": "Province"
+    },
+    {
+      "code": "MZ-T",
+      "name": "Tete",
+      "type": "Province"
+    },
+    {
+      "code": "NA-CA",
+      "name": "Zambezi",
+      "type": "Region"
+    },
+    {
+      "code": "NA-ER",
+      "name": "Erongo",
+      "type": "Region"
+    },
+    {
+      "code": "NA-HA",
+      "name": "Hardap",
+      "type": "Region"
+    },
+    {
+      "code": "NA-KA",
+      "name": "//Karas",
+      "type": "Region"
+    },
+    {
+      "code": "NA-KE",
+      "name": "Kavango East",
+      "type": "Region"
+    },
+    {
+      "code": "NA-KH",
+      "name": "Khomas",
+      "type": "Region"
+    },
+    {
+      "code": "NA-KU",
+      "name": "Kunene",
+      "type": "Region"
+    },
+    {
+      "code": "NA-KW",
+      "name": "Kavango West",
+      "type": "Region"
+    },
+    {
+      "code": "NA-OD",
+      "name": "Otjozondjupa",
+      "type": "Region"
+    },
+    {
+      "code": "NA-OH",
+      "name": "Omaheke",
+      "type": "Region"
+    },
+    {
+      "code": "NA-ON",
+      "name": "Oshana",
+      "type": "Region"
+    },
+    {
+      "code": "NA-OS",
+      "name": "Omusati",
+      "type": "Region"
+    },
+    {
+      "code": "NA-OT",
+      "name": "Oshikoto",
+      "type": "Region"
+    },
+    {
+      "code": "NA-OW",
+      "name": "Ohangwena",
+      "type": "Region"
+    },
+    {
+      "code": "NE-1",
+      "name": "Agadez",
+      "type": "Region"
+    },
+    {
+      "code": "NE-2",
+      "name": "Diffa",
+      "type": "Region"
+    },
+    {
+      "code": "NE-3",
+      "name": "Dosso",
+      "type": "Region"
+    },
+    {
+      "code": "NE-4",
+      "name": "Maradi",
+      "type": "Region"
+    },
+    {
+      "code": "NE-5",
+      "name": "Tahoua",
+      "type": "Region"
+    },
+    {
+      "code": "NE-6",
+      "name": "Tillabéri",
+      "type": "Region"
+    },
+    {
+      "code": "NE-7",
+      "name": "Zinder",
+      "type": "Region"
+    },
+    {
+      "code": "NE-8",
+      "name": "Niamey",
+      "type": "Urban community"
+    },
+    {
+      "code": "NG-AB",
+      "name": "Abia",
+      "type": "State"
+    },
+    {
+      "code": "NG-AD",
+      "name": "Adamawa",
+      "type": "State"
+    },
+    {
+      "code": "NG-AK",
+      "name": "Akwa Ibom",
+      "type": "State"
+    },
+    {
+      "code": "NG-AN",
+      "name": "Anambra",
+      "type": "State"
+    },
+    {
+      "code": "NG-BA",
+      "name": "Bauchi",
+      "type": "State"
+    },
+    {
+      "code": "NG-BE",
+      "name": "Benue",
+      "type": "State"
+    },
+    {
+      "code": "NG-BO",
+      "name": "Borno",
+      "type": "State"
+    },
+    {
+      "code": "NG-BY",
+      "name": "Bayelsa",
+      "type": "State"
+    },
+    {
+      "code": "NG-CR",
+      "name": "Cross River",
+      "type": "State"
+    },
+    {
+      "code": "NG-DE",
+      "name": "Delta",
+      "type": "State"
+    },
+    {
+      "code": "NG-EB",
+      "name": "Ebonyi",
+      "type": "State"
+    },
+    {
+      "code": "NG-ED",
+      "name": "Edo",
+      "type": "State"
+    },
+    {
+      "code": "NG-EK",
+      "name": "Ekiti",
+      "type": "State"
+    },
+    {
+      "code": "NG-EN",
+      "name": "Enugu",
+      "type": "State"
+    },
+    {
+      "code": "NG-FC",
+      "name": "Abuja Federal Capital Territory",
+      "type": "Capital territory"
+    },
+    {
+      "code": "NG-GO",
+      "name": "Gombe",
+      "type": "State"
+    },
+    {
+      "code": "NG-IM",
+      "name": "Imo",
+      "type": "State"
+    },
+    {
+      "code": "NG-JI",
+      "name": "Jigawa",
+      "type": "State"
+    },
+    {
+      "code": "NG-KD",
+      "name": "Kaduna",
+      "type": "State"
+    },
+    {
+      "code": "NG-KE",
+      "name": "Kebbi",
+      "type": "State"
+    },
+    {
+      "code": "NG-KN",
+      "name": "Kano",
+      "type": "State"
+    },
+    {
+      "code": "NG-KO",
+      "name": "Kogi",
+      "type": "State"
+    },
+    {
+      "code": "NG-KT",
+      "name": "Katsina",
+      "type": "State"
+    },
+    {
+      "code": "NG-KW",
+      "name": "Kwara",
+      "type": "State"
+    },
+    {
+      "code": "NG-LA",
+      "name": "Lagos",
+      "type": "State"
+    },
+    {
+      "code": "NG-NA",
+      "name": "Nasarawa",
+      "type": "State"
+    },
+    {
+      "code": "NG-NI",
+      "name": "Niger",
+      "type": "State"
+    },
+    {
+      "code": "NG-OG",
+      "name": "Ogun",
+      "type": "State"
+    },
+    {
+      "code": "NG-ON",
+      "name": "Ondo",
+      "type": "State"
+    },
+    {
+      "code": "NG-OS",
+      "name": "Osun",
+      "type": "State"
+    },
+    {
+      "code": "NG-OY",
+      "name": "Oyo",
+      "type": "State"
+    },
+    {
+      "code": "NG-PL",
+      "name": "Plateau",
+      "type": "State"
+    },
+    {
+      "code": "NG-RI",
+      "name": "Rivers",
+      "type": "State"
+    },
+    {
+      "code": "NG-SO",
+      "name": "Sokoto",
+      "type": "State"
+    },
+    {
+      "code": "NG-TA",
+      "name": "Taraba",
+      "type": "State"
+    },
+    {
+      "code": "NG-YO",
+      "name": "Yobe",
+      "type": "State"
+    },
+    {
+      "code": "NG-ZA",
+      "name": "Zamfara",
+      "type": "State"
+    },
+    {
+      "code": "NI-AN",
+      "name": "Costa Caribe Norte",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "NI-AS",
+      "name": "Costa Caribe Sur",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "NI-BO",
+      "name": "Boaco",
+      "type": "Department"
+    },
+    {
+      "code": "NI-CA",
+      "name": "Carazo",
+      "type": "Department"
+    },
+    {
+      "code": "NI-CI",
+      "name": "Chinandega",
+      "type": "Department"
+    },
+    {
+      "code": "NI-CO",
+      "name": "Chontales",
+      "type": "Department"
+    },
+    {
+      "code": "NI-ES",
+      "name": "Estelí",
+      "type": "Department"
+    },
+    {
+      "code": "NI-GR",
+      "name": "Granada",
+      "type": "Department"
+    },
+    {
+      "code": "NI-JI",
+      "name": "Jinotega",
+      "type": "Department"
+    },
+    {
+      "code": "NI-LE",
+      "name": "León",
+      "type": "Department"
+    },
+    {
+      "code": "NI-MD",
+      "name": "Madriz",
+      "type": "Department"
+    },
+    {
+      "code": "NI-MN",
+      "name": "Managua",
+      "type": "Department"
+    },
+    {
+      "code": "NI-MS",
+      "name": "Masaya",
+      "type": "Department"
+    },
+    {
+      "code": "NI-MT",
+      "name": "Matagalpa",
+      "type": "Department"
+    },
+    {
+      "code": "NI-NS",
+      "name": "Nueva Segovia",
+      "type": "Department"
+    },
+    {
+      "code": "NI-RI",
+      "name": "Rivas",
+      "type": "Department"
+    },
+    {
+      "code": "NI-SJ",
+      "name": "Río San Juan",
+      "type": "Department"
+    },
+    {
+      "code": "NL-AW",
+      "name": "Aruba",
+      "type": "Country"
+    },
+    {
+      "code": "NL-BQ1",
+      "name": "Bonaire",
+      "type": "Special municipality"
+    },
+    {
+      "code": "NL-BQ2",
+      "name": "Saba",
+      "type": "Special municipality"
+    },
+    {
+      "code": "NL-BQ3",
+      "name": "Sint Eustatius",
+      "type": "Special municipality"
+    },
+    {
+      "code": "NL-CW",
+      "name": "Curaçao",
+      "type": "Country"
+    },
+    {
+      "code": "NL-DR",
+      "name": "Drenthe",
+      "type": "Province"
+    },
+    {
+      "code": "NL-FL",
+      "name": "Flevoland",
+      "type": "Province"
+    },
+    {
+      "code": "NL-FR",
+      "name": "Fryslân",
+      "type": "Province"
+    },
+    {
+      "code": "NL-GE",
+      "name": "Gelderland",
+      "type": "Province"
+    },
+    {
+      "code": "NL-GR",
+      "name": "Groningen",
+      "type": "Province"
+    },
+    {
+      "code": "NL-LI",
+      "name": "Limburg",
+      "type": "Province"
+    },
+    {
+      "code": "NL-NB",
+      "name": "Noord-Brabant",
+      "type": "Province"
+    },
+    {
+      "code": "NL-NH",
+      "name": "Noord-Holland",
+      "type": "Province"
+    },
+    {
+      "code": "NL-OV",
+      "name": "Overijssel",
+      "type": "Province"
+    },
+    {
+      "code": "NL-SX",
+      "name": "Sint Maarten",
+      "type": "Country"
+    },
+    {
+      "code": "NL-UT",
+      "name": "Utrecht",
+      "type": "Province"
+    },
+    {
+      "code": "NL-ZE",
+      "name": "Zeeland",
+      "type": "Province"
+    },
+    {
+      "code": "NL-ZH",
+      "name": "Zuid-Holland",
+      "type": "Province"
+    },
+    {
+      "code": "NO-03",
+      "name": "Oslo",
+      "type": "County"
+    },
+    {
+      "code": "NO-11",
+      "name": "Rogaland",
+      "type": "County"
+    },
+    {
+      "code": "NO-15",
+      "name": "Møre og Romsdal",
+      "type": "County"
+    },
+    {
+      "code": "NO-18",
+      "name": "Nordland",
+      "type": "County"
+    },
+    {
+      "code": "NO-21",
+      "name": "Svalbard (Arctic Region)",
+      "type": "Arctic region"
+    },
+    {
+      "code": "NO-22",
+      "name": "Jan Mayen (Arctic Region)",
+      "type": "Arctic region"
+    },
+    {
+      "code": "NO-30",
+      "name": "Viken",
+      "type": "County"
+    },
+    {
+      "code": "NO-34",
+      "name": "Innlandet",
+      "type": "County"
+    },
+    {
+      "code": "NO-38",
+      "name": "Vestfold og Telemark",
+      "type": "County"
+    },
+    {
+      "code": "NO-42",
+      "name": "Agder",
+      "type": "County"
+    },
+    {
+      "code": "NO-46",
+      "name": "Vestland",
+      "type": "County"
+    },
+    {
+      "code": "NO-50",
+      "name": "Trööndelage",
+      "type": "County"
+    },
+    {
+      "code": "NO-54",
+      "name": "Romssa ja Finnmárkku",
+      "type": "County"
+    },
+    {
+      "code": "NP-P1",
+      "name": "Koshi",
+      "type": "Province"
+    },
+    {
+      "code": "NP-P2",
+      "name": "Madhesh",
+      "type": "Province"
+    },
+    {
+      "code": "NP-P3",
+      "name": "Bagmati",
+      "type": "Province"
+    },
+    {
+      "code": "NP-P4",
+      "name": "Gandaki",
+      "type": "Province"
+    },
+    {
+      "code": "NP-P5",
+      "name": "Lumbini",
+      "type": "Province"
+    },
+    {
+      "code": "NP-P6",
+      "name": "Karnali",
+      "type": "Province"
+    },
+    {
+      "code": "NP-P7",
+      "name": "Sudurpashchim",
+      "type": "Province"
+    },
+    {
+      "code": "NR-01",
+      "name": "Aiwo",
+      "type": "District"
+    },
+    {
+      "code": "NR-02",
+      "name": "Anabar",
+      "type": "District"
+    },
+    {
+      "code": "NR-03",
+      "name": "Anetan",
+      "type": "District"
+    },
+    {
+      "code": "NR-04",
+      "name": "Anibare",
+      "type": "District"
+    },
+    {
+      "code": "NR-05",
+      "name": "Baitsi",
+      "type": "District"
+    },
+    {
+      "code": "NR-06",
+      "name": "Boe",
+      "type": "District"
+    },
+    {
+      "code": "NR-07",
+      "name": "Buada",
+      "type": "District"
+    },
+    {
+      "code": "NR-08",
+      "name": "Denigomodu",
+      "type": "District"
+    },
+    {
+      "code": "NR-09",
+      "name": "Ewa",
+      "type": "District"
+    },
+    {
+      "code": "NR-10",
+      "name": "Ijuw",
+      "type": "District"
+    },
+    {
+      "code": "NR-11",
+      "name": "Meneng",
+      "type": "District"
+    },
+    {
+      "code": "NR-12",
+      "name": "Nibok",
+      "type": "District"
+    },
+    {
+      "code": "NR-13",
+      "name": "Uaboe",
+      "type": "District"
+    },
+    {
+      "code": "NR-14",
+      "name": "Yaren",
+      "type": "District"
+    },
+    {
+      "code": "NZ-AUK",
+      "name": "Auckland",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-BOP",
+      "name": "Bay of Plenty",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-CAN",
+      "name": "Canterbury",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-CIT",
+      "name": "Chatham Islands Territory",
+      "type": "Special island authority"
+    },
+    {
+      "code": "NZ-GIS",
+      "name": "Gisborne",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-HKB",
+      "name": "Hawke's Bay",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-MBH",
+      "name": "Marlborough",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-MWT",
+      "name": "Manawatū-Whanganui",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-NSN",
+      "name": "Nelson",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-NTL",
+      "name": "Northland",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-OTA",
+      "name": "Otago",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-STL",
+      "name": "Southland",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-TAS",
+      "name": "Tasman",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-TKI",
+      "name": "Taranaki",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-WGN",
+      "name": "Greater Wellington",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-WKO",
+      "name": "Waikato",
+      "type": "Region"
+    },
+    {
+      "code": "NZ-WTC",
+      "name": "West Coast",
+      "type": "Region"
+    },
+    {
+      "code": "OM-BJ",
+      "name": "Janūb al Bāţinah",
+      "type": "Governorate"
+    },
+    {
+      "code": "OM-BS",
+      "name": "Shamāl al Bāţinah",
+      "type": "Governorate"
+    },
+    {
+      "code": "OM-BU",
+      "name": "Al Buraymī",
+      "type": "Governorate"
+    },
+    {
+      "code": "OM-DA",
+      "name": "Ad Dākhilīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "OM-MA",
+      "name": "Masqaţ",
+      "type": "Governorate"
+    },
+    {
+      "code": "OM-MU",
+      "name": "Musandam",
+      "type": "Governorate"
+    },
+    {
+      "code": "OM-SJ",
+      "name": "Janūb ash Sharqīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "OM-SS",
+      "name": "Shamāl ash Sharqīyah",
+      "type": "Governorate"
+    },
+    {
+      "code": "OM-WU",
+      "name": "Al Wusţá",
+      "type": "Governorate"
+    },
+    {
+      "code": "OM-ZA",
+      "name": "Az̧ Z̧āhirah",
+      "type": "Governorate"
+    },
+    {
+      "code": "OM-ZU",
+      "name": "Z̧ufār",
+      "type": "Governorate"
+    },
+    {
+      "code": "PA-1",
+      "name": "Bocas del Toro",
+      "type": "Province"
+    },
+    {
+      "code": "PA-10",
+      "name": "Panamá Oeste",
+      "type": "Province"
+    },
+    {
+      "code": "PA-2",
+      "name": "Coclé",
+      "type": "Province"
+    },
+    {
+      "code": "PA-3",
+      "name": "Colón",
+      "type": "Province"
+    },
+    {
+      "code": "PA-4",
+      "name": "Chiriquí",
+      "type": "Province"
+    },
+    {
+      "code": "PA-5",
+      "name": "Darién",
+      "type": "Province"
+    },
+    {
+      "code": "PA-6",
+      "name": "Herrera",
+      "type": "Province"
+    },
+    {
+      "code": "PA-7",
+      "name": "Los Santos",
+      "type": "Province"
+    },
+    {
+      "code": "PA-8",
+      "name": "Panamá",
+      "type": "Province"
+    },
+    {
+      "code": "PA-9",
+      "name": "Veraguas",
+      "type": "Province"
+    },
+    {
+      "code": "PA-EM",
+      "name": "Emberá",
+      "type": "Indigenous region"
+    },
+    {
+      "code": "PA-KY",
+      "name": "Guna Yala",
+      "type": "Indigenous region"
+    },
+    {
+      "code": "PA-NB",
+      "name": "Ngäbe-Buglé",
+      "type": "Indigenous region"
+    },
+    {
+      "code": "PA-NT",
+      "name": "Naso Tjër Di",
+      "type": "Indigenous region"
+    },
+    {
+      "code": "PE-AMA",
+      "name": "Amarumayu",
+      "type": "Region"
+    },
+    {
+      "code": "PE-ANC",
+      "name": "Ancash",
+      "type": "Region"
+    },
+    {
+      "code": "PE-APU",
+      "name": "Apurimaq",
+      "type": "Region"
+    },
+    {
+      "code": "PE-ARE",
+      "name": "Arequipa",
+      "type": "Region"
+    },
+    {
+      "code": "PE-AYA",
+      "name": "Ayacucho",
+      "type": "Region"
+    },
+    {
+      "code": "PE-CAJ",
+      "name": "Cajamarca",
+      "type": "Region"
+    },
+    {
+      "code": "PE-CAL",
+      "name": "El Callao",
+      "type": "Region"
+    },
+    {
+      "code": "PE-CUS",
+      "name": "Cusco",
+      "type": "Region"
+    },
+    {
+      "code": "PE-HUC",
+      "name": "Huánuco",
+      "type": "Region"
+    },
+    {
+      "code": "PE-HUV",
+      "name": "Huancavelica",
+      "type": "Region"
+    },
+    {
+      "code": "PE-ICA",
+      "name": "Ica",
+      "type": "Region"
+    },
+    {
+      "code": "PE-JUN",
+      "name": "Hunin",
+      "type": "Region"
+    },
+    {
+      "code": "PE-LAL",
+      "name": "La Libertad",
+      "type": "Region"
+    },
+    {
+      "code": "PE-LAM",
+      "name": "Lambayeque",
+      "type": "Region"
+    },
+    {
+      "code": "PE-LIM",
+      "name": "Lima",
+      "type": "Region"
+    },
+    {
+      "code": "PE-LMA",
+      "name": "Lima hatun llaqta",
+      "type": "Municipality"
+    },
+    {
+      "code": "PE-LOR",
+      "name": "Loreto",
+      "type": "Region"
+    },
+    {
+      "code": "PE-MDD",
+      "name": "Madre de Dios",
+      "type": "Region"
+    },
+    {
+      "code": "PE-MOQ",
+      "name": "Moquegua",
+      "type": "Region"
+    },
+    {
+      "code": "PE-PAS",
+      "name": "Pasco",
+      "type": "Region"
+    },
+    {
+      "code": "PE-PIU",
+      "name": "Piura",
+      "type": "Region"
+    },
+    {
+      "code": "PE-PUN",
+      "name": "Puno",
+      "type": "Region"
+    },
+    {
+      "code": "PE-SAM",
+      "name": "San Martin",
+      "type": "Region"
+    },
+    {
+      "code": "PE-TAC",
+      "name": "Tacna",
+      "type": "Region"
+    },
+    {
+      "code": "PE-TUM",
+      "name": "Tumbes",
+      "type": "Region"
+    },
+    {
+      "code": "PE-UCA",
+      "name": "Ucayali",
+      "type": "Region"
+    },
+    {
+      "code": "PG-CPK",
+      "name": "Chimbu",
+      "type": "Province"
+    },
+    {
+      "code": "PG-CPM",
+      "name": "Central",
+      "type": "Province"
+    },
+    {
+      "code": "PG-EBR",
+      "name": "East New Britain",
+      "type": "Province"
+    },
+    {
+      "code": "PG-EHG",
+      "name": "Eastern Highlands",
+      "type": "Province"
+    },
+    {
+      "code": "PG-EPW",
+      "name": "Enga",
+      "type": "Province"
+    },
+    {
+      "code": "PG-ESW",
+      "name": "East Sepik",
+      "type": "Province"
+    },
+    {
+      "code": "PG-GPK",
+      "name": "Gulf",
+      "type": "Province"
+    },
+    {
+      "code": "PG-HLA",
+      "name": "Hela",
+      "type": "Province"
+    },
+    {
+      "code": "PG-JWK",
+      "name": "Jiwaka",
+      "type": "Province"
+    },
+    {
+      "code": "PG-MBA",
+      "name": "Milne Bay",
+      "type": "Province"
+    },
+    {
+      "code": "PG-MPL",
+      "name": "Morobe",
+      "type": "Province"
+    },
+    {
+      "code": "PG-MPM",
+      "name": "Madang",
+      "type": "Province"
+    },
+    {
+      "code": "PG-MRL",
+      "name": "Manus",
+      "type": "Province"
+    },
+    {
+      "code": "PG-NCD",
+      "name": "National Capital District (Port Moresby)",
+      "type": "District"
+    },
+    {
+      "code": "PG-NIK",
+      "name": "New Ireland",
+      "type": "Province"
+    },
+    {
+      "code": "PG-NPP",
+      "name": "Northern",
+      "type": "Province"
+    },
+    {
+      "code": "PG-NSB",
+      "name": "Bougainville",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "PG-SAN",
+      "name": "West Sepik",
+      "type": "Province"
+    },
+    {
+      "code": "PG-SHM",
+      "name": "Southern Highlands",
+      "type": "Province"
+    },
+    {
+      "code": "PG-WBK",
+      "name": "West New Britain",
+      "type": "Province"
+    },
+    {
+      "code": "PG-WHM",
+      "name": "Western Highlands",
+      "type": "Province"
+    },
+    {
+      "code": "PG-WPD",
+      "name": "Western",
+      "type": "Province"
+    },
+    {
+      "code": "PH-00",
+      "name": "National Capital Region",
+      "type": "Region"
+    },
+    {
+      "code": "PH-01",
+      "name": "Ilocos (Region I)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-02",
+      "name": "Cagayan Valley (Region II)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-03",
+      "name": "Central Luzon (Region III)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-05",
+      "name": "Bicol (Region V)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-06",
+      "name": "Western Visayas (Region VI)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-07",
+      "name": "Central Visayas (Region VII)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-08",
+      "name": "Eastern Visayas (Region VIII)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-09",
+      "name": "Zamboanga Peninsula (Region IX)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-10",
+      "name": "Northern Mindanao (Region X)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-11",
+      "name": "Davao (Region XI)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-12",
+      "name": "Soccsksargen (Region XII)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-13",
+      "name": "Caraga (Region XIII)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-14",
+      "name": "Autonomous Region in Muslim Mindanao (ARMM)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-15",
+      "name": "Cordillera Administrative Region (CAR)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-40",
+      "name": "Calabarzon (Region IV-A)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-41",
+      "name": "Mimaropa (Region IV-B)",
+      "type": "Region"
+    },
+    {
+      "code": "PH-ABR",
+      "name": "Abra",
+      "parent": "PH-15",
+      "type": "Province"
+    },
+    {
+      "code": "PH-AGN",
+      "name": "Agusan del Norte",
+      "parent": "PH-13",
+      "type": "Province"
+    },
+    {
+      "code": "PH-AGS",
+      "name": "Agusan del Sur",
+      "parent": "PH-13",
+      "type": "Province"
+    },
+    {
+      "code": "PH-AKL",
+      "name": "Aklan",
+      "parent": "PH-06",
+      "type": "Province"
+    },
+    {
+      "code": "PH-ALB",
+      "name": "Albay",
+      "parent": "PH-05",
+      "type": "Province"
+    },
+    {
+      "code": "PH-ANT",
+      "name": "Antique",
+      "parent": "PH-06",
+      "type": "Province"
+    },
+    {
+      "code": "PH-APA",
+      "name": "Apayao",
+      "parent": "PH-15",
+      "type": "Province"
+    },
+    {
+      "code": "PH-AUR",
+      "name": "Aurora",
+      "parent": "PH-03",
+      "type": "Province"
+    },
+    {
+      "code": "PH-BAN",
+      "name": "Bataan",
+      "parent": "PH-03",
+      "type": "Province"
+    },
+    {
+      "code": "PH-BAS",
+      "name": "Basilan",
+      "parent": "PH-09",
+      "type": "Province"
+    },
+    {
+      "code": "PH-BEN",
+      "name": "Benguet",
+      "parent": "PH-15",
+      "type": "Province"
+    },
+    {
+      "code": "PH-BIL",
+      "name": "Biliran",
+      "parent": "PH-08",
+      "type": "Province"
+    },
+    {
+      "code": "PH-BOH",
+      "name": "Bohol",
+      "parent": "PH-07",
+      "type": "Province"
+    },
+    {
+      "code": "PH-BTG",
+      "name": "Batangas",
+      "parent": "PH-40",
+      "type": "Province"
+    },
+    {
+      "code": "PH-BTN",
+      "name": "Batanes",
+      "parent": "PH-02",
+      "type": "Province"
+    },
+    {
+      "code": "PH-BUK",
+      "name": "Bukidnon",
+      "parent": "PH-10",
+      "type": "Province"
+    },
+    {
+      "code": "PH-BUL",
+      "name": "Bulacan",
+      "parent": "PH-03",
+      "type": "Province"
+    },
+    {
+      "code": "PH-CAG",
+      "name": "Cagayan",
+      "parent": "PH-02",
+      "type": "Province"
+    },
+    {
+      "code": "PH-CAM",
+      "name": "Camiguin",
+      "parent": "PH-10",
+      "type": "Province"
+    },
+    {
+      "code": "PH-CAN",
+      "name": "Camarines Norte",
+      "parent": "PH-05",
+      "type": "Province"
+    },
+    {
+      "code": "PH-CAP",
+      "name": "Capiz",
+      "parent": "PH-06",
+      "type": "Province"
+    },
+    {
+      "code": "PH-CAS",
+      "name": "Camarines Sur",
+      "parent": "PH-05",
+      "type": "Province"
+    },
+    {
+      "code": "PH-CAT",
+      "name": "Catanduanes",
+      "parent": "PH-05",
+      "type": "Province"
+    },
+    {
+      "code": "PH-CAV",
+      "name": "Cavite",
+      "parent": "PH-40",
+      "type": "Province"
+    },
+    {
+      "code": "PH-CEB",
+      "name": "Cebu",
+      "parent": "PH-07",
+      "type": "Province"
+    },
+    {
+      "code": "PH-COM",
+      "name": "Davao de Oro",
+      "parent": "PH-11",
+      "type": "Province"
+    },
+    {
+      "code": "PH-DAO",
+      "name": "Davao Oriental",
+      "parent": "PH-11",
+      "type": "Province"
+    },
+    {
+      "code": "PH-DAS",
+      "name": "Davao del Sur",
+      "parent": "PH-11",
+      "type": "Province"
+    },
+    {
+      "code": "PH-DAV",
+      "name": "Davao del Norte",
+      "parent": "PH-11",
+      "type": "Province"
+    },
+    {
+      "code": "PH-DIN",
+      "name": "Dinagat Islands",
+      "parent": "PH-13",
+      "type": "Province"
+    },
+    {
+      "code": "PH-DVO",
+      "name": "Davao Occidental",
+      "parent": "PH-11",
+      "type": "Province"
+    },
+    {
+      "code": "PH-EAS",
+      "name": "Eastern Samar",
+      "parent": "PH-08",
+      "type": "Province"
+    },
+    {
+      "code": "PH-GUI",
+      "name": "Guimaras",
+      "parent": "PH-06",
+      "type": "Province"
+    },
+    {
+      "code": "PH-IFU",
+      "name": "Ifugao",
+      "parent": "PH-15",
+      "type": "Province"
+    },
+    {
+      "code": "PH-ILI",
+      "name": "Iloilo",
+      "parent": "PH-06",
+      "type": "Province"
+    },
+    {
+      "code": "PH-ILN",
+      "name": "Ilocos Norte",
+      "parent": "PH-01",
+      "type": "Province"
+    },
+    {
+      "code": "PH-ILS",
+      "name": "Ilocos Sur",
+      "parent": "PH-01",
+      "type": "Province"
+    },
+    {
+      "code": "PH-ISA",
+      "name": "Isabela",
+      "parent": "PH-02",
+      "type": "Province"
+    },
+    {
+      "code": "PH-KAL",
+      "name": "Kalinga",
+      "parent": "PH-15",
+      "type": "Province"
+    },
+    {
+      "code": "PH-LAG",
+      "name": "Laguna",
+      "parent": "PH-40",
+      "type": "Province"
+    },
+    {
+      "code": "PH-LAN",
+      "name": "Lanao del Norte",
+      "parent": "PH-12",
+      "type": "Province"
+    },
+    {
+      "code": "PH-LAS",
+      "name": "Lanao del Sur",
+      "parent": "PH-14",
+      "type": "Province"
+    },
+    {
+      "code": "PH-LEY",
+      "name": "Leyte",
+      "parent": "PH-08",
+      "type": "Province"
+    },
+    {
+      "code": "PH-LUN",
+      "name": "La Union",
+      "parent": "PH-01",
+      "type": "Province"
+    },
+    {
+      "code": "PH-MAD",
+      "name": "Marinduque",
+      "parent": "PH-41",
+      "type": "Province"
+    },
+    {
+      "code": "PH-MAS",
+      "name": "Masbate",
+      "parent": "PH-05",
+      "type": "Province"
+    },
+    {
+      "code": "PH-MDC",
+      "name": "Mindoro Occidental",
+      "parent": "PH-41",
+      "type": "Province"
+    },
+    {
+      "code": "PH-MDR",
+      "name": "Mindoro Oriental",
+      "parent": "PH-41",
+      "type": "Province"
+    },
+    {
+      "code": "PH-MGN",
+      "name": "Maguindanao del Norte",
+      "parent": "PH-14",
+      "type": "Province"
+    },
+    {
+      "code": "PH-MGS",
+      "name": "Maguindanao del Sur",
+      "parent": "PH-14",
+      "type": "Province"
+    },
+    {
+      "code": "PH-MOU",
+      "name": "Mountain Province",
+      "parent": "PH-15",
+      "type": "Province"
+    },
+    {
+      "code": "PH-MSC",
+      "name": "Misamis Occidental",
+      "parent": "PH-10",
+      "type": "Province"
+    },
+    {
+      "code": "PH-MSR",
+      "name": "Misamis Oriental",
+      "parent": "PH-10",
+      "type": "Province"
+    },
+    {
+      "code": "PH-NCO",
+      "name": "Cotabato",
+      "parent": "PH-12",
+      "type": "Province"
+    },
+    {
+      "code": "PH-NEC",
+      "name": "Negros Occidental",
+      "parent": "PH-06",
+      "type": "Province"
+    },
+    {
+      "code": "PH-NER",
+      "name": "Negros Oriental",
+      "parent": "PH-07",
+      "type": "Province"
+    },
+    {
+      "code": "PH-NSA",
+      "name": "Northern Samar",
+      "parent": "PH-08",
+      "type": "Province"
+    },
+    {
+      "code": "PH-NUE",
+      "name": "Nueva Ecija",
+      "parent": "PH-03",
+      "type": "Province"
+    },
+    {
+      "code": "PH-NUV",
+      "name": "Nueva Vizcaya",
+      "parent": "PH-02",
+      "type": "Province"
+    },
+    {
+      "code": "PH-PAM",
+      "name": "Pampanga",
+      "parent": "PH-03",
+      "type": "Province"
+    },
+    {
+      "code": "PH-PAN",
+      "name": "Pangasinan",
+      "parent": "PH-01",
+      "type": "Province"
+    },
+    {
+      "code": "PH-PLW",
+      "name": "Palawan",
+      "parent": "PH-41",
+      "type": "Province"
+    },
+    {
+      "code": "PH-QUE",
+      "name": "Quezon",
+      "parent": "PH-40",
+      "type": "Province"
+    },
+    {
+      "code": "PH-QUI",
+      "name": "Quirino",
+      "parent": "PH-02",
+      "type": "Province"
+    },
+    {
+      "code": "PH-RIZ",
+      "name": "Rizal",
+      "parent": "PH-40",
+      "type": "Province"
+    },
+    {
+      "code": "PH-ROM",
+      "name": "Romblon",
+      "parent": "PH-41",
+      "type": "Province"
+    },
+    {
+      "code": "PH-SAR",
+      "name": "Sarangani",
+      "parent": "PH-11",
+      "type": "Province"
+    },
+    {
+      "code": "PH-SCO",
+      "name": "South Cotabato",
+      "parent": "PH-11",
+      "type": "Province"
+    },
+    {
+      "code": "PH-SIG",
+      "name": "Siquijor",
+      "parent": "PH-07",
+      "type": "Province"
+    },
+    {
+      "code": "PH-SLE",
+      "name": "Southern Leyte",
+      "parent": "PH-08",
+      "type": "Province"
+    },
+    {
+      "code": "PH-SLU",
+      "name": "Sulu",
+      "parent": "PH-14",
+      "type": "Province"
+    },
+    {
+      "code": "PH-SOR",
+      "name": "Sorsogon",
+      "parent": "PH-05",
+      "type": "Province"
+    },
+    {
+      "code": "PH-SUK",
+      "name": "Sultan Kudarat",
+      "parent": "PH-12",
+      "type": "Province"
+    },
+    {
+      "code": "PH-SUN",
+      "name": "Surigao del Norte",
+      "parent": "PH-13",
+      "type": "Province"
+    },
+    {
+      "code": "PH-SUR",
+      "name": "Surigao del Sur",
+      "parent": "PH-13",
+      "type": "Province"
+    },
+    {
+      "code": "PH-TAR",
+      "name": "Tarlac",
+      "parent": "PH-03",
+      "type": "Province"
+    },
+    {
+      "code": "PH-TAW",
+      "name": "Tawi-Tawi",
+      "parent": "PH-14",
+      "type": "Province"
+    },
+    {
+      "code": "PH-WSA",
+      "name": "Samar",
+      "parent": "PH-08",
+      "type": "Province"
+    },
+    {
+      "code": "PH-ZAN",
+      "name": "Zamboanga del Norte",
+      "parent": "PH-09",
+      "type": "Province"
+    },
+    {
+      "code": "PH-ZAS",
+      "name": "Zamboanga del Sur",
+      "parent": "PH-09",
+      "type": "Province"
+    },
+    {
+      "code": "PH-ZMB",
+      "name": "Zambales",
+      "parent": "PH-03",
+      "type": "Province"
+    },
+    {
+      "code": "PH-ZSI",
+      "name": "Zamboanga Sibugay",
+      "parent": "PH-09",
+      "type": "Province"
+    },
+    {
+      "code": "PK-BA",
+      "name": "Balochistan",
+      "type": "Province"
+    },
+    {
+      "code": "PK-GB",
+      "name": "Gilgit-Baltistan",
+      "type": "Pakistan administered area"
+    },
+    {
+      "code": "PK-IS",
+      "name": "Islamabad",
+      "type": "Federal capital territory"
+    },
+    {
+      "code": "PK-JK",
+      "name": "Azad Jammu and Kashmir",
+      "type": "Pakistan administered area"
+    },
+    {
+      "code": "PK-KP",
+      "name": "Khyber Pakhtunkhwa",
+      "type": "Province"
+    },
+    {
+      "code": "PK-PB",
+      "name": "Punjab",
+      "type": "Province"
+    },
+    {
+      "code": "PK-SD",
+      "name": "Sindh",
+      "type": "Province"
+    },
+    {
+      "code": "PL-02",
+      "name": "Dolnośląskie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-04",
+      "name": "Kujawsko-Pomorskie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-06",
+      "name": "Lubelskie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-08",
+      "name": "Lubuskie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-10",
+      "name": "Łódzkie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-12",
+      "name": "Małopolskie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-14",
+      "name": "Mazowieckie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-16",
+      "name": "Opolskie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-18",
+      "name": "Podkarpackie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-20",
+      "name": "Podlaskie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-22",
+      "name": "Pomorskie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-24",
+      "name": "Śląskie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-26",
+      "name": "Świętokrzyskie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-28",
+      "name": "Warmińsko-Mazurskie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-30",
+      "name": "Wielkopolskie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PL-32",
+      "name": "Zachodniopomorskie",
+      "type": "Voivodship"
+    },
+    {
+      "code": "PS-BTH",
+      "name": "Bethlehem",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-DEB",
+      "name": "Deir El Balah",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-GZA",
+      "name": "Gaza",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-HBN",
+      "name": "Hebron",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-JEM",
+      "name": "Jerusalem",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-JEN",
+      "name": "Jenin",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-JRH",
+      "name": "Jericho and Al Aghwar",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-KYS",
+      "name": "Khan Yunis",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-NBS",
+      "name": "Nablus",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-NGZ",
+      "name": "North Gaza",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-QQA",
+      "name": "Qalqilya",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-RBH",
+      "name": "Ramallah",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-RFH",
+      "name": "Rafah",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-SLT",
+      "name": "Salfit",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-TBS",
+      "name": "Tubas",
+      "type": "Governorate"
+    },
+    {
+      "code": "PS-TKM",
+      "name": "Tulkarm",
+      "type": "Governorate"
+    },
+    {
+      "code": "PT-01",
+      "name": "Aveiro",
+      "type": "District"
+    },
+    {
+      "code": "PT-02",
+      "name": "Beja",
+      "type": "District"
+    },
+    {
+      "code": "PT-03",
+      "name": "Braga",
+      "type": "District"
+    },
+    {
+      "code": "PT-04",
+      "name": "Bragança",
+      "type": "District"
+    },
+    {
+      "code": "PT-05",
+      "name": "Castelo Branco",
+      "type": "District"
+    },
+    {
+      "code": "PT-06",
+      "name": "Coimbra",
+      "type": "District"
+    },
+    {
+      "code": "PT-07",
+      "name": "Évora",
+      "type": "District"
+    },
+    {
+      "code": "PT-08",
+      "name": "Faro",
+      "type": "District"
+    },
+    {
+      "code": "PT-09",
+      "name": "Guarda",
+      "type": "District"
+    },
+    {
+      "code": "PT-10",
+      "name": "Leiria",
+      "type": "District"
+    },
+    {
+      "code": "PT-11",
+      "name": "Lisboa",
+      "type": "District"
+    },
+    {
+      "code": "PT-12",
+      "name": "Portalegre",
+      "type": "District"
+    },
+    {
+      "code": "PT-13",
+      "name": "Porto",
+      "type": "District"
+    },
+    {
+      "code": "PT-14",
+      "name": "Santarém",
+      "type": "District"
+    },
+    {
+      "code": "PT-15",
+      "name": "Setúbal",
+      "type": "District"
+    },
+    {
+      "code": "PT-16",
+      "name": "Viana do Castelo",
+      "type": "District"
+    },
+    {
+      "code": "PT-17",
+      "name": "Vila Real",
+      "type": "District"
+    },
+    {
+      "code": "PT-18",
+      "name": "Viseu",
+      "type": "District"
+    },
+    {
+      "code": "PT-20",
+      "name": "Região Autónoma dos Açores",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "PT-30",
+      "name": "Região Autónoma da Madeira",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "PW-002",
+      "name": "Aimeliik",
+      "type": "State"
+    },
+    {
+      "code": "PW-004",
+      "name": "Airai",
+      "type": "State"
+    },
+    {
+      "code": "PW-010",
+      "name": "Angaur",
+      "type": "State"
+    },
+    {
+      "code": "PW-050",
+      "name": "Hatohobei",
+      "type": "State"
+    },
+    {
+      "code": "PW-100",
+      "name": "Kayangel",
+      "type": "State"
+    },
+    {
+      "code": "PW-150",
+      "name": "Koror",
+      "type": "State"
+    },
+    {
+      "code": "PW-212",
+      "name": "Melekeok",
+      "type": "State"
+    },
+    {
+      "code": "PW-214",
+      "name": "Ngaraard",
+      "type": "State"
+    },
+    {
+      "code": "PW-218",
+      "name": "Ngarchelong",
+      "type": "State"
+    },
+    {
+      "code": "PW-222",
+      "name": "Ngardmau",
+      "type": "State"
+    },
+    {
+      "code": "PW-224",
+      "name": "Ngatpang",
+      "type": "State"
+    },
+    {
+      "code": "PW-226",
+      "name": "Ngchesar",
+      "type": "State"
+    },
+    {
+      "code": "PW-227",
+      "name": "Ngeremlengui",
+      "type": "State"
+    },
+    {
+      "code": "PW-228",
+      "name": "Ngiwal",
+      "type": "State"
+    },
+    {
+      "code": "PW-350",
+      "name": "Peleliu",
+      "type": "State"
+    },
+    {
+      "code": "PW-370",
+      "name": "Sonsorol",
+      "type": "State"
+    },
+    {
+      "code": "PY-1",
+      "name": "Concepción",
+      "type": "Department"
+    },
+    {
+      "code": "PY-10",
+      "name": "Alto Paraná",
+      "type": "Department"
+    },
+    {
+      "code": "PY-11",
+      "name": "Central",
+      "type": "Department"
+    },
+    {
+      "code": "PY-12",
+      "name": "Ñeembucú",
+      "type": "Department"
+    },
+    {
+      "code": "PY-13",
+      "name": "Amambay",
+      "type": "Department"
+    },
+    {
+      "code": "PY-14",
+      "name": "Canindeyú",
+      "type": "Department"
+    },
+    {
+      "code": "PY-15",
+      "name": "Presidente Hayes",
+      "type": "Department"
+    },
+    {
+      "code": "PY-16",
+      "name": "Alto Paraguay",
+      "type": "Department"
+    },
+    {
+      "code": "PY-19",
+      "name": "Boquerón",
+      "type": "Department"
+    },
+    {
+      "code": "PY-2",
+      "name": "San Pedro",
+      "type": "Department"
+    },
+    {
+      "code": "PY-3",
+      "name": "Cordillera",
+      "type": "Department"
+    },
+    {
+      "code": "PY-4",
+      "name": "Guairá",
+      "type": "Department"
+    },
+    {
+      "code": "PY-5",
+      "name": "Caaguazú",
+      "type": "Department"
+    },
+    {
+      "code": "PY-6",
+      "name": "Caazapá",
+      "type": "Department"
+    },
+    {
+      "code": "PY-7",
+      "name": "Itapúa",
+      "type": "Department"
+    },
+    {
+      "code": "PY-8",
+      "name": "Misiones",
+      "type": "Department"
+    },
+    {
+      "code": "PY-9",
+      "name": "Paraguarí",
+      "type": "Department"
+    },
+    {
+      "code": "PY-ASU",
+      "name": "Asunción",
+      "type": "Capital"
+    },
+    {
+      "code": "QA-DA",
+      "name": "Ad Dawḩah",
+      "type": "Municipality"
+    },
+    {
+      "code": "QA-KH",
+      "name": "Al Khawr wa adh Dhakhīrah",
+      "type": "Municipality"
+    },
+    {
+      "code": "QA-MS",
+      "name": "Ash Shamāl",
+      "type": "Municipality"
+    },
+    {
+      "code": "QA-RA",
+      "name": "Ar Rayyān",
+      "type": "Municipality"
+    },
+    {
+      "code": "QA-SH",
+      "name": "Ash Shīḩānīyah",
+      "type": "Municipality"
+    },
+    {
+      "code": "QA-US",
+      "name": "Umm Şalāl",
+      "type": "Municipality"
+    },
+    {
+      "code": "QA-WA",
+      "name": "Al Wakrah",
+      "type": "Municipality"
+    },
+    {
+      "code": "QA-ZA",
+      "name": "Az̧ Z̧a‘āyin",
+      "type": "Municipality"
+    },
+    {
+      "code": "RO-AB",
+      "name": "Alba",
+      "type": "Department"
+    },
+    {
+      "code": "RO-AG",
+      "name": "Argeș",
+      "type": "Department"
+    },
+    {
+      "code": "RO-AR",
+      "name": "Arad",
+      "type": "Department"
+    },
+    {
+      "code": "RO-B",
+      "name": "București",
+      "type": "Municipality"
+    },
+    {
+      "code": "RO-BC",
+      "name": "Bacău",
+      "type": "Department"
+    },
+    {
+      "code": "RO-BH",
+      "name": "Bihor",
+      "type": "Department"
+    },
+    {
+      "code": "RO-BN",
+      "name": "Bistrița-Năsăud",
+      "type": "Department"
+    },
+    {
+      "code": "RO-BR",
+      "name": "Brăila",
+      "type": "Department"
+    },
+    {
+      "code": "RO-BT",
+      "name": "Botoșani",
+      "type": "Department"
+    },
+    {
+      "code": "RO-BV",
+      "name": "Brașov",
+      "type": "Department"
+    },
+    {
+      "code": "RO-BZ",
+      "name": "Buzău",
+      "type": "Department"
+    },
+    {
+      "code": "RO-CJ",
+      "name": "Cluj",
+      "type": "Department"
+    },
+    {
+      "code": "RO-CL",
+      "name": "Călărași",
+      "type": "Department"
+    },
+    {
+      "code": "RO-CS",
+      "name": "Caraș-Severin",
+      "type": "Department"
+    },
+    {
+      "code": "RO-CT",
+      "name": "Constanța",
+      "type": "Department"
+    },
+    {
+      "code": "RO-CV",
+      "name": "Covasna",
+      "type": "Department"
+    },
+    {
+      "code": "RO-DB",
+      "name": "Dâmbovița",
+      "type": "Department"
+    },
+    {
+      "code": "RO-DJ",
+      "name": "Dolj",
+      "type": "Department"
+    },
+    {
+      "code": "RO-GJ",
+      "name": "Gorj",
+      "type": "Department"
+    },
+    {
+      "code": "RO-GL",
+      "name": "Galați",
+      "type": "Department"
+    },
+    {
+      "code": "RO-GR",
+      "name": "Giurgiu",
+      "type": "Department"
+    },
+    {
+      "code": "RO-HD",
+      "name": "Hunedoara",
+      "type": "Department"
+    },
+    {
+      "code": "RO-HR",
+      "name": "Harghita",
+      "type": "Department"
+    },
+    {
+      "code": "RO-IF",
+      "name": "Ilfov",
+      "type": "Department"
+    },
+    {
+      "code": "RO-IL",
+      "name": "Ialomița",
+      "type": "Department"
+    },
+    {
+      "code": "RO-IS",
+      "name": "Iași",
+      "type": "Department"
+    },
+    {
+      "code": "RO-MH",
+      "name": "Mehedinți",
+      "type": "Department"
+    },
+    {
+      "code": "RO-MM",
+      "name": "Maramureș",
+      "type": "Department"
+    },
+    {
+      "code": "RO-MS",
+      "name": "Mureș",
+      "type": "Department"
+    },
+    {
+      "code": "RO-NT",
+      "name": "Neamț",
+      "type": "Department"
+    },
+    {
+      "code": "RO-OT",
+      "name": "Olt",
+      "type": "Department"
+    },
+    {
+      "code": "RO-PH",
+      "name": "Prahova",
+      "type": "Department"
+    },
+    {
+      "code": "RO-SB",
+      "name": "Sibiu",
+      "type": "Department"
+    },
+    {
+      "code": "RO-SJ",
+      "name": "Sălaj",
+      "type": "Department"
+    },
+    {
+      "code": "RO-SM",
+      "name": "Satu Mare",
+      "type": "Department"
+    },
+    {
+      "code": "RO-SV",
+      "name": "Suceava",
+      "type": "Department"
+    },
+    {
+      "code": "RO-TL",
+      "name": "Tulcea",
+      "type": "Department"
+    },
+    {
+      "code": "RO-TM",
+      "name": "Timiș",
+      "type": "Department"
+    },
+    {
+      "code": "RO-TR",
+      "name": "Teleorman",
+      "type": "Department"
+    },
+    {
+      "code": "RO-VL",
+      "name": "Vâlcea",
+      "type": "Department"
+    },
+    {
+      "code": "RO-VN",
+      "name": "Vrancea",
+      "type": "Department"
+    },
+    {
+      "code": "RO-VS",
+      "name": "Vaslui",
+      "type": "Department"
+    },
+    {
+      "code": "RS-00",
+      "name": "Beograd",
+      "type": "City"
+    },
+    {
+      "code": "RS-01",
+      "name": "Severnobački okrug",
+      "parent": "RS-VO",
+      "type": "District"
+    },
+    {
+      "code": "RS-02",
+      "name": "Srednjebanatski okrug",
+      "parent": "RS-VO",
+      "type": "District"
+    },
+    {
+      "code": "RS-03",
+      "name": "Severnobanatski okrug",
+      "parent": "RS-VO",
+      "type": "District"
+    },
+    {
+      "code": "RS-04",
+      "name": "Južnobanatski okrug",
+      "parent": "RS-VO",
+      "type": "District"
+    },
+    {
+      "code": "RS-05",
+      "name": "Zapadnobački okrug",
+      "parent": "RS-VO",
+      "type": "District"
+    },
+    {
+      "code": "RS-06",
+      "name": "Južnobački okrug",
+      "parent": "RS-VO",
+      "type": "District"
+    },
+    {
+      "code": "RS-07",
+      "name": "Sremski okrug",
+      "parent": "RS-VO",
+      "type": "District"
+    },
+    {
+      "code": "RS-08",
+      "name": "Mačvanski okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-09",
+      "name": "Kolubarski okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-10",
+      "name": "Podunavski okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-11",
+      "name": "Braničevski okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-12",
+      "name": "Šumadijski okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-13",
+      "name": "Pomoravski okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-14",
+      "name": "Borski okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-15",
+      "name": "Zaječarski okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-16",
+      "name": "Zlatiborski okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-17",
+      "name": "Moravički okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-18",
+      "name": "Raški okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-19",
+      "name": "Rasinski okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-20",
+      "name": "Nišavski okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-21",
+      "name": "Toplički okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-22",
+      "name": "Pirotski okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-23",
+      "name": "Jablanički okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-24",
+      "name": "Pčinjski okrug",
+      "type": "District"
+    },
+    {
+      "code": "RS-25",
+      "name": "Kosovski okrug",
+      "parent": "RS-KM",
+      "type": "District"
+    },
+    {
+      "code": "RS-26",
+      "name": "Pećki okrug",
+      "parent": "RS-KM",
+      "type": "District"
+    },
+    {
+      "code": "RS-27",
+      "name": "Prizrenski okrug",
+      "parent": "RS-KM",
+      "type": "District"
+    },
+    {
+      "code": "RS-28",
+      "name": "Kosovsko-Mitrovački okrug",
+      "parent": "RS-KM",
+      "type": "District"
+    },
+    {
+      "code": "RS-29",
+      "name": "Kosovsko-Pomoravski okrug",
+      "parent": "RS-KM",
+      "type": "District"
+    },
+    {
+      "code": "RS-KM",
+      "name": "Kosovo-Metohija",
+      "type": "Autonomous province"
+    },
+    {
+      "code": "RS-VO",
+      "name": "Vojvodina",
+      "type": "Autonomous province"
+    },
+    {
+      "code": "RU-AD",
+      "name": "Adygeja, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-AL",
+      "name": "Altaj, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-ALT",
+      "name": "Altajskij kraj",
+      "type": "Administrative territory"
+    },
+    {
+      "code": "RU-AMU",
+      "name": "Amurskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-ARK",
+      "name": "Arhangel'skaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-AST",
+      "name": "Astrahanskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-BA",
+      "name": "Bashkortostan, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-BEL",
+      "name": "Belgorodskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-BRY",
+      "name": "Brjanskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-BU",
+      "name": "Burjatija, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-CE",
+      "name": "Chechenskaya Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-CHE",
+      "name": "Chelyabinskaya oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-CHU",
+      "name": "Chukotskiy avtonomnyy okrug",
+      "type": "Autonomous district"
+    },
+    {
+      "code": "RU-CU",
+      "name": "Chuvashskaya Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-DA",
+      "name": "Dagestan, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-IN",
+      "name": "Ingushetiya, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-IRK",
+      "name": "Irkutskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-IVA",
+      "name": "Ivanovskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-KAM",
+      "name": "Kamchatskiy kray",
+      "type": "Administrative territory"
+    },
+    {
+      "code": "RU-KB",
+      "name": "Kabardino-Balkarskaja Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-KC",
+      "name": "Karachayevo-Cherkesskaya Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-KDA",
+      "name": "Krasnodarskij kraj",
+      "type": "Administrative territory"
+    },
+    {
+      "code": "RU-KEM",
+      "name": "Kemerovskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-KGD",
+      "name": "Kaliningradskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-KGN",
+      "name": "Kurganskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-KHA",
+      "name": "Habarovskij kraj",
+      "type": "Administrative territory"
+    },
+    {
+      "code": "RU-KHM",
+      "name": "Hanty-Mansijskij avtonomnyj okrug",
+      "type": "Autonomous district"
+    },
+    {
+      "code": "RU-KIR",
+      "name": "Kirovskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-KK",
+      "name": "Hakasija, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-KL",
+      "name": "Kalmykija, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-KLU",
+      "name": "Kaluzhskaya oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-KO",
+      "name": "Komi, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-KOS",
+      "name": "Kostromskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-KR",
+      "name": "Karelija, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-KRS",
+      "name": "Kurskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-KYA",
+      "name": "Krasnojarskij kraj",
+      "type": "Administrative territory"
+    },
+    {
+      "code": "RU-LEN",
+      "name": "Leningradskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-LIP",
+      "name": "Lipeckaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-MAG",
+      "name": "Magadanskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-ME",
+      "name": "Marij Èl, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-MO",
+      "name": "Mordovija, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-MOS",
+      "name": "Moskovskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-MOW",
+      "name": "Moskva",
+      "type": "Autonomous city"
+    },
+    {
+      "code": "RU-MUR",
+      "name": "Murmanskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-NEN",
+      "name": "Neneckij avtonomnyj okrug",
+      "type": "Autonomous district"
+    },
+    {
+      "code": "RU-NGR",
+      "name": "Novgorodskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-NIZ",
+      "name": "Nizhegorodskaya oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-NVS",
+      "name": "Novosibirskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-OMS",
+      "name": "Omskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-ORE",
+      "name": "Orenburgskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-ORL",
+      "name": "Orlovskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-PER",
+      "name": "Permskij kraj",
+      "type": "Administrative territory"
+    },
+    {
+      "code": "RU-PNZ",
+      "name": "Penzenskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-PRI",
+      "name": "Primorskij kraj",
+      "type": "Administrative territory"
+    },
+    {
+      "code": "RU-PSK",
+      "name": "Pskovskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-ROS",
+      "name": "Rostovskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-RYA",
+      "name": "Rjazanskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-SA",
+      "name": "Saha, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-SAK",
+      "name": "Sahalinskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-SAM",
+      "name": "Samarskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-SAR",
+      "name": "Saratovskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-SE",
+      "name": "Severnaja Osetija, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-SMO",
+      "name": "Smolenskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-SPE",
+      "name": "Sankt-Peterburg",
+      "type": "Autonomous city"
+    },
+    {
+      "code": "RU-STA",
+      "name": "Stavropol'skij kraj",
+      "type": "Administrative territory"
+    },
+    {
+      "code": "RU-SVE",
+      "name": "Sverdlovskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-TA",
+      "name": "Tatarstan, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-TAM",
+      "name": "Tambovskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-TOM",
+      "name": "Tomskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-TUL",
+      "name": "Tul'skaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-TVE",
+      "name": "Tverskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-TY",
+      "name": "Tyva, Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-TYU",
+      "name": "Tjumenskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-UD",
+      "name": "Udmurtskaja Respublika",
+      "type": "Republic"
+    },
+    {
+      "code": "RU-ULY",
+      "name": "Ul'janovskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-VGG",
+      "name": "Volgogradskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-VLA",
+      "name": "Vladimirskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-VLG",
+      "name": "Vologodskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-VOR",
+      "name": "Voronezhskaya oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-YAN",
+      "name": "Jamalo-Neneckij avtonomnyj okrug",
+      "type": "Autonomous district"
+    },
+    {
+      "code": "RU-YAR",
+      "name": "Jaroslavskaja oblast'",
+      "type": "Administrative region"
+    },
+    {
+      "code": "RU-YEV",
+      "name": "Evrejskaja avtonomnaja oblast'",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "RU-ZAB",
+      "name": "Zabajkal'skij kraj",
+      "type": "Administrative territory"
+    },
+    {
+      "code": "RW-01",
+      "name": "City of Kigali",
+      "type": "City"
+    },
+    {
+      "code": "RW-02",
+      "name": "Eastern",
+      "type": "Province"
+    },
+    {
+      "code": "RW-03",
+      "name": "Northern",
+      "type": "Province"
+    },
+    {
+      "code": "RW-04",
+      "name": "Western",
+      "type": "Province"
+    },
+    {
+      "code": "RW-05",
+      "name": "Southern",
+      "type": "Province"
+    },
+    {
+      "code": "SA-01",
+      "name": "Ar Riyāḑ",
+      "type": "Region"
+    },
+    {
+      "code": "SA-02",
+      "name": "Makkah al Mukarramah",
+      "type": "Region"
+    },
+    {
+      "code": "SA-03",
+      "name": "Al Madīnah al Munawwarah",
+      "type": "Region"
+    },
+    {
+      "code": "SA-04",
+      "name": "Ash Sharqīyah",
+      "type": "Region"
+    },
+    {
+      "code": "SA-05",
+      "name": "Al Qaşīm",
+      "type": "Region"
+    },
+    {
+      "code": "SA-06",
+      "name": "Ḩā'il",
+      "type": "Region"
+    },
+    {
+      "code": "SA-07",
+      "name": "Tabūk",
+      "type": "Region"
+    },
+    {
+      "code": "SA-08",
+      "name": "Al Ḩudūd ash Shamālīyah",
+      "type": "Region"
+    },
+    {
+      "code": "SA-09",
+      "name": "Jāzān",
+      "type": "Region"
+    },
+    {
+      "code": "SA-10",
+      "name": "Najrān",
+      "type": "Region"
+    },
+    {
+      "code": "SA-11",
+      "name": "Al Bāḩah",
+      "type": "Region"
+    },
+    {
+      "code": "SA-12",
+      "name": "Al Jawf",
+      "type": "Region"
+    },
+    {
+      "code": "SA-14",
+      "name": "'Asīr",
+      "type": "Region"
+    },
+    {
+      "code": "SB-CE",
+      "name": "Central",
+      "type": "Province"
+    },
+    {
+      "code": "SB-CH",
+      "name": "Choiseul",
+      "type": "Province"
+    },
+    {
+      "code": "SB-CT",
+      "name": "Capital Territory (Honiara)",
+      "type": "Capital territory"
+    },
+    {
+      "code": "SB-GU",
+      "name": "Guadalcanal",
+      "type": "Province"
+    },
+    {
+      "code": "SB-IS",
+      "name": "Isabel",
+      "type": "Province"
+    },
+    {
+      "code": "SB-MK",
+      "name": "Makira-Ulawa",
+      "type": "Province"
+    },
+    {
+      "code": "SB-ML",
+      "name": "Malaita",
+      "type": "Province"
+    },
+    {
+      "code": "SB-RB",
+      "name": "Rennell and Bellona",
+      "type": "Province"
+    },
+    {
+      "code": "SB-TE",
+      "name": "Temotu",
+      "type": "Province"
+    },
+    {
+      "code": "SB-WE",
+      "name": "Western",
+      "type": "Province"
+    },
+    {
+      "code": "SC-01",
+      "name": "Anse aux Pins",
+      "type": "District"
+    },
+    {
+      "code": "SC-02",
+      "name": "Anse Boileau",
+      "type": "District"
+    },
+    {
+      "code": "SC-03",
+      "name": "Anse Etoile",
+      "type": "District"
+    },
+    {
+      "code": "SC-04",
+      "name": "Au Cap",
+      "type": "District"
+    },
+    {
+      "code": "SC-05",
+      "name": "Anse Royale",
+      "type": "District"
+    },
+    {
+      "code": "SC-06",
+      "name": "Baie Lazare",
+      "type": "District"
+    },
+    {
+      "code": "SC-07",
+      "name": "Baie Sainte Anne",
+      "type": "District"
+    },
+    {
+      "code": "SC-08",
+      "name": "Beau Vallon",
+      "type": "District"
+    },
+    {
+      "code": "SC-09",
+      "name": "Bel Air",
+      "type": "District"
+    },
+    {
+      "code": "SC-10",
+      "name": "Bel Ombre",
+      "type": "District"
+    },
+    {
+      "code": "SC-11",
+      "name": "Cascade",
+      "type": "District"
+    },
+    {
+      "code": "SC-12",
+      "name": "Glacis",
+      "type": "District"
+    },
+    {
+      "code": "SC-13",
+      "name": "Grand Anse Mahe",
+      "type": "District"
+    },
+    {
+      "code": "SC-14",
+      "name": "Grand Anse Praslin",
+      "type": "District"
+    },
+    {
+      "code": "SC-15",
+      "name": "La Digue",
+      "type": "District"
+    },
+    {
+      "code": "SC-16",
+      "name": "English River",
+      "type": "District"
+    },
+    {
+      "code": "SC-17",
+      "name": "Mont Buxton",
+      "type": "District"
+    },
+    {
+      "code": "SC-18",
+      "name": "Mont Fleuri",
+      "type": "District"
+    },
+    {
+      "code": "SC-19",
+      "name": "Plaisance",
+      "type": "District"
+    },
+    {
+      "code": "SC-20",
+      "name": "Pointe Larue",
+      "type": "District"
+    },
+    {
+      "code": "SC-21",
+      "name": "Port Glaud",
+      "type": "District"
+    },
+    {
+      "code": "SC-22",
+      "name": "Saint Louis",
+      "type": "District"
+    },
+    {
+      "code": "SC-23",
+      "name": "Takamaka",
+      "type": "District"
+    },
+    {
+      "code": "SC-24",
+      "name": "Les Mamelles",
+      "type": "District"
+    },
+    {
+      "code": "SC-25",
+      "name": "Roche Caiman",
+      "type": "District"
+    },
+    {
+      "code": "SC-26",
+      "name": "Ile Perseverance I",
+      "type": "District"
+    },
+    {
+      "code": "SC-27",
+      "name": "Ile Perseverance II",
+      "type": "District"
+    },
+    {
+      "code": "SD-DC",
+      "name": "Central Darfur",
+      "type": "State"
+    },
+    {
+      "code": "SD-DE",
+      "name": "East Darfur",
+      "type": "State"
+    },
+    {
+      "code": "SD-DN",
+      "name": "North Darfur",
+      "type": "State"
+    },
+    {
+      "code": "SD-DS",
+      "name": "South Darfur",
+      "type": "State"
+    },
+    {
+      "code": "SD-DW",
+      "name": "West Darfur",
+      "type": "State"
+    },
+    {
+      "code": "SD-GD",
+      "name": "Gedaref",
+      "type": "State"
+    },
+    {
+      "code": "SD-GK",
+      "name": "West Kordofan",
+      "type": "State"
+    },
+    {
+      "code": "SD-GZ",
+      "name": "Gezira",
+      "type": "State"
+    },
+    {
+      "code": "SD-KA",
+      "name": "Kassala",
+      "type": "State"
+    },
+    {
+      "code": "SD-KH",
+      "name": "Khartoum",
+      "type": "State"
+    },
+    {
+      "code": "SD-KN",
+      "name": "North Kordofan",
+      "type": "State"
+    },
+    {
+      "code": "SD-KS",
+      "name": "South Kordofan",
+      "type": "State"
+    },
+    {
+      "code": "SD-NB",
+      "name": "Blue Nile",
+      "type": "State"
+    },
+    {
+      "code": "SD-NO",
+      "name": "Northern",
+      "type": "State"
+    },
+    {
+      "code": "SD-NR",
+      "name": "River Nile",
+      "type": "State"
+    },
+    {
+      "code": "SD-NW",
+      "name": "White Nile",
+      "type": "State"
+    },
+    {
+      "code": "SD-RS",
+      "name": "Red Sea",
+      "type": "State"
+    },
+    {
+      "code": "SD-SI",
+      "name": "Sennar",
+      "type": "State"
+    },
+    {
+      "code": "SE-AB",
+      "name": "Stockholms län [SE-01]",
+      "type": "County"
+    },
+    {
+      "code": "SE-AC",
+      "name": "Västerbottens län [SE-24]",
+      "type": "County"
+    },
+    {
+      "code": "SE-BD",
+      "name": "Norrbottens län [SE-25]",
+      "type": "County"
+    },
+    {
+      "code": "SE-C",
+      "name": "Uppsala län [SE-03]",
+      "type": "County"
+    },
+    {
+      "code": "SE-D",
+      "name": "Södermanlands län [SE-04]",
+      "type": "County"
+    },
+    {
+      "code": "SE-E",
+      "name": "Östergötlands län [SE-05]",
+      "type": "County"
+    },
+    {
+      "code": "SE-F",
+      "name": "Jönköpings län [SE-06]",
+      "type": "County"
+    },
+    {
+      "code": "SE-G",
+      "name": "Kronobergs län [SE-07]",
+      "type": "County"
+    },
+    {
+      "code": "SE-H",
+      "name": "Kalmar län [SE-08]",
+      "type": "County"
+    },
+    {
+      "code": "SE-I",
+      "name": "Gotlands län [SE-09]",
+      "type": "County"
+    },
+    {
+      "code": "SE-K",
+      "name": "Blekinge län [SE-10]",
+      "type": "County"
+    },
+    {
+      "code": "SE-M",
+      "name": "Skåne län [SE-12]",
+      "type": "County"
+    },
+    {
+      "code": "SE-N",
+      "name": "Hallands län [SE-13]",
+      "type": "County"
+    },
+    {
+      "code": "SE-O",
+      "name": "Västra Götalands län [SE-14]",
+      "type": "County"
+    },
+    {
+      "code": "SE-S",
+      "name": "Värmlands län [SE-17]",
+      "type": "County"
+    },
+    {
+      "code": "SE-T",
+      "name": "Örebro län [SE-18]",
+      "type": "County"
+    },
+    {
+      "code": "SE-U",
+      "name": "Västmanlands län [SE-19]",
+      "type": "County"
+    },
+    {
+      "code": "SE-W",
+      "name": "Dalarnas län [SE-20]",
+      "type": "County"
+    },
+    {
+      "code": "SE-X",
+      "name": "Gävleborgs län [SE-21]",
+      "type": "County"
+    },
+    {
+      "code": "SE-Y",
+      "name": "Västernorrlands län [SE-22]",
+      "type": "County"
+    },
+    {
+      "code": "SE-Z",
+      "name": "Jämtlands län [SE-23]",
+      "type": "County"
+    },
+    {
+      "code": "SG-01",
+      "name": "Central Singapore",
+      "type": "District"
+    },
+    {
+      "code": "SG-02",
+      "name": "North East",
+      "type": "District"
+    },
+    {
+      "code": "SG-03",
+      "name": "North West",
+      "type": "District"
+    },
+    {
+      "code": "SG-04",
+      "name": "South East",
+      "type": "District"
+    },
+    {
+      "code": "SG-05",
+      "name": "South West",
+      "type": "District"
+    },
+    {
+      "code": "SH-AC",
+      "name": "Ascension",
+      "type": "Geographical entity"
+    },
+    {
+      "code": "SH-HL",
+      "name": "Saint Helena",
+      "type": "Geographical entity"
+    },
+    {
+      "code": "SH-TA",
+      "name": "Tristan da Cunha",
+      "type": "Geographical entity"
+    },
+    {
+      "code": "SI-001",
+      "name": "Ajdovščina",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-002",
+      "name": "Beltinci",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-003",
+      "name": "Bled",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-004",
+      "name": "Bohinj",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-005",
+      "name": "Borovnica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-006",
+      "name": "Bovec",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-007",
+      "name": "Brda",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-008",
+      "name": "Brezovica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-009",
+      "name": "Brežice",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-010",
+      "name": "Tišina",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-011",
+      "name": "Celje",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "SI-012",
+      "name": "Cerklje na Gorenjskem",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-013",
+      "name": "Cerknica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-014",
+      "name": "Cerkno",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-015",
+      "name": "Črenšovci",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-016",
+      "name": "Črna na Koroškem",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-017",
+      "name": "Črnomelj",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-018",
+      "name": "Destrnik",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-019",
+      "name": "Divača",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-020",
+      "name": "Dobrepolje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-021",
+      "name": "Dobrova-Polhov Gradec",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-022",
+      "name": "Dol pri Ljubljani",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-023",
+      "name": "Domžale",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-024",
+      "name": "Dornava",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-025",
+      "name": "Dravograd",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-026",
+      "name": "Duplek",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-027",
+      "name": "Gorenja vas-Poljane",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-028",
+      "name": "Gorišnica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-029",
+      "name": "Gornja Radgona",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-030",
+      "name": "Gornji Grad",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-031",
+      "name": "Gornji Petrovci",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-032",
+      "name": "Grosuplje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-033",
+      "name": "Šalovci",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-034",
+      "name": "Hrastnik",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-035",
+      "name": "Hrpelje-Kozina",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-036",
+      "name": "Idrija",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-037",
+      "name": "Ig",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-038",
+      "name": "Ilirska Bistrica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-039",
+      "name": "Ivančna Gorica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-040",
+      "name": "Izola",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-041",
+      "name": "Jesenice",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-042",
+      "name": "Juršinci",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-043",
+      "name": "Kamnik",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-044",
+      "name": "Kanal ob Soči",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-045",
+      "name": "Kidričevo",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-046",
+      "name": "Kobarid",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-047",
+      "name": "Kobilje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-048",
+      "name": "Kočevje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-049",
+      "name": "Komen",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-050",
+      "name": "Koper",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "SI-051",
+      "name": "Kozje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-052",
+      "name": "Kranj",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "SI-053",
+      "name": "Kranjska Gora",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-054",
+      "name": "Krško",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "SI-055",
+      "name": "Kungota",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-056",
+      "name": "Kuzma",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-057",
+      "name": "Laško",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-058",
+      "name": "Lenart",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-059",
+      "name": "Lendava",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-060",
+      "name": "Litija",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-061",
+      "name": "Ljubljana",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "SI-062",
+      "name": "Ljubno",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-063",
+      "name": "Ljutomer",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-064",
+      "name": "Logatec",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-065",
+      "name": "Loška dolina",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-066",
+      "name": "Loški Potok",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-067",
+      "name": "Luče",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-068",
+      "name": "Lukovica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-069",
+      "name": "Majšperk",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-070",
+      "name": "Maribor",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "SI-071",
+      "name": "Medvode",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-072",
+      "name": "Mengeš",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-073",
+      "name": "Metlika",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-074",
+      "name": "Mežica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-075",
+      "name": "Miren-Kostanjevica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-076",
+      "name": "Mislinja",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-077",
+      "name": "Moravče",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-078",
+      "name": "Moravske Toplice",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-079",
+      "name": "Mozirje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-080",
+      "name": "Murska Sobota",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "SI-081",
+      "name": "Muta",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-082",
+      "name": "Naklo",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-083",
+      "name": "Nazarje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-084",
+      "name": "Nova Gorica",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "SI-085",
+      "name": "Novo Mesto",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "SI-086",
+      "name": "Odranci",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-087",
+      "name": "Ormož",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-088",
+      "name": "Osilnica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-089",
+      "name": "Pesnica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-090",
+      "name": "Piran",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-091",
+      "name": "Pivka",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-092",
+      "name": "Podčetrtek",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-093",
+      "name": "Podvelka",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-094",
+      "name": "Postojna",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-095",
+      "name": "Preddvor",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-096",
+      "name": "Ptuj",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "SI-097",
+      "name": "Puconci",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-098",
+      "name": "Rače-Fram",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-099",
+      "name": "Radeče",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-100",
+      "name": "Radenci",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-101",
+      "name": "Radlje ob Dravi",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-102",
+      "name": "Radovljica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-103",
+      "name": "Ravne na Koroškem",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-104",
+      "name": "Ribnica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-105",
+      "name": "Rogašovci",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-106",
+      "name": "Rogaška Slatina",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-107",
+      "name": "Rogatec",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-108",
+      "name": "Ruše",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-109",
+      "name": "Semič",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-110",
+      "name": "Sevnica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-111",
+      "name": "Sežana",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-112",
+      "name": "Slovenj Gradec",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "SI-113",
+      "name": "Slovenska Bistrica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-114",
+      "name": "Slovenske Konjice",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-115",
+      "name": "Starše",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-116",
+      "name": "Sveti Jurij ob Ščavnici",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-117",
+      "name": "Šenčur",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-118",
+      "name": "Šentilj",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-119",
+      "name": "Šentjernej",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-120",
+      "name": "Šentjur",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-121",
+      "name": "Škocjan",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-122",
+      "name": "Škofja Loka",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-123",
+      "name": "Škofljica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-124",
+      "name": "Šmarje pri Jelšah",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-125",
+      "name": "Šmartno ob Paki",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-126",
+      "name": "Šoštanj",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-127",
+      "name": "Štore",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-128",
+      "name": "Tolmin",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-129",
+      "name": "Trbovlje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-130",
+      "name": "Trebnje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-131",
+      "name": "Tržič",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-132",
+      "name": "Turnišče",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-133",
+      "name": "Velenje",
+      "type": "Urban municipality"
+    },
+    {
+      "code": "SI-134",
+      "name": "Velike Lašče",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-135",
+      "name": "Videm",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-136",
+      "name": "Vipava",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-137",
+      "name": "Vitanje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-138",
+      "name": "Vodice",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-139",
+      "name": "Vojnik",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-140",
+      "name": "Vrhnika",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-141",
+      "name": "Vuzenica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-142",
+      "name": "Zagorje ob Savi",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-143",
+      "name": "Zavrč",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-144",
+      "name": "Zreče",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-146",
+      "name": "Železniki",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-147",
+      "name": "Žiri",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-148",
+      "name": "Benedikt",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-149",
+      "name": "Bistrica ob Sotli",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-150",
+      "name": "Bloke",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-151",
+      "name": "Braslovče",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-152",
+      "name": "Cankova",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-153",
+      "name": "Cerkvenjak",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-154",
+      "name": "Dobje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-155",
+      "name": "Dobrna",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-156",
+      "name": "Dobrovnik",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-157",
+      "name": "Dolenjske Toplice",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-158",
+      "name": "Grad",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-159",
+      "name": "Hajdina",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-160",
+      "name": "Hoče-Slivnica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-161",
+      "name": "Hodoš",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-162",
+      "name": "Horjul",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-163",
+      "name": "Jezersko",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-164",
+      "name": "Komenda",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-165",
+      "name": "Kostel",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-166",
+      "name": "Križevci",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-167",
+      "name": "Lovrenc na Pohorju",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-168",
+      "name": "Markovci",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-169",
+      "name": "Miklavž na Dravskem polju",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-170",
+      "name": "Mirna Peč",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-171",
+      "name": "Oplotnica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-172",
+      "name": "Podlehnik",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-173",
+      "name": "Polzela",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-174",
+      "name": "Prebold",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-175",
+      "name": "Prevalje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-176",
+      "name": "Razkrižje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-177",
+      "name": "Ribnica na Pohorju",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-178",
+      "name": "Selnica ob Dravi",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-179",
+      "name": "Sodražica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-180",
+      "name": "Solčava",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-181",
+      "name": "Sveta Ana",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-182",
+      "name": "Sveti Andraž v Slovenskih goricah",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-183",
+      "name": "Šempeter-Vrtojba",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-184",
+      "name": "Tabor",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-185",
+      "name": "Trnovska Vas",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-186",
+      "name": "Trzin",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-187",
+      "name": "Velika Polana",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-188",
+      "name": "Veržej",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-189",
+      "name": "Vransko",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-190",
+      "name": "Žalec",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-191",
+      "name": "Žetale",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-192",
+      "name": "Žirovnica",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-193",
+      "name": "Žužemberk",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-194",
+      "name": "Šmartno pri Litiji",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-195",
+      "name": "Apače",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-196",
+      "name": "Cirkulane",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-197",
+      "name": "Kostanjevica na Krki",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-198",
+      "name": "Makole",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-199",
+      "name": "Mokronog-Trebelno",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-200",
+      "name": "Poljčane",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-201",
+      "name": "Renče-Vogrsko",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-202",
+      "name": "Središče ob Dravi",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-203",
+      "name": "Straža",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-204",
+      "name": "Sveta Trojica v Slovenskih goricah",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-205",
+      "name": "Sveti Tomaž",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-206",
+      "name": "Šmarješke Toplice",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-207",
+      "name": "Gorje",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-208",
+      "name": "Log-Dragomer",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-209",
+      "name": "Rečica ob Savinji",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-210",
+      "name": "Sveti Jurij v Slovenskih goricah",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-211",
+      "name": "Šentrupert",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-212",
+      "name": "Mirna",
+      "type": "Municipality"
+    },
+    {
+      "code": "SI-213",
+      "name": "Ankaran",
+      "type": "Municipality"
+    },
+    {
+      "code": "SK-BC",
+      "name": "Banskobystrický kraj",
+      "type": "Region"
+    },
+    {
+      "code": "SK-BL",
+      "name": "Bratislavský kraj",
+      "type": "Region"
+    },
+    {
+      "code": "SK-KI",
+      "name": "Košický kraj",
+      "type": "Region"
+    },
+    {
+      "code": "SK-NI",
+      "name": "Nitriansky kraj",
+      "type": "Region"
+    },
+    {
+      "code": "SK-PV",
+      "name": "Prešovský kraj",
+      "type": "Region"
+    },
+    {
+      "code": "SK-TA",
+      "name": "Trnavský kraj",
+      "type": "Region"
+    },
+    {
+      "code": "SK-TC",
+      "name": "Trenčiansky kraj",
+      "type": "Region"
+    },
+    {
+      "code": "SK-ZI",
+      "name": "Žilinský kraj",
+      "type": "Region"
+    },
+    {
+      "code": "SL-E",
+      "name": "Eastern",
+      "type": "Province"
+    },
+    {
+      "code": "SL-N",
+      "name": "Northern",
+      "type": "Province"
+    },
+    {
+      "code": "SL-NW",
+      "name": "North Western",
+      "type": "Province"
+    },
+    {
+      "code": "SL-S",
+      "name": "Southern",
+      "type": "Province"
+    },
+    {
+      "code": "SL-W",
+      "name": "Western Area (Freetown)",
+      "type": "Area"
+    },
+    {
+      "code": "SM-01",
+      "name": "Acquaviva",
+      "type": "Municipality"
+    },
+    {
+      "code": "SM-02",
+      "name": "Chiesanuova",
+      "type": "Municipality"
+    },
+    {
+      "code": "SM-03",
+      "name": "Domagnano",
+      "type": "Municipality"
+    },
+    {
+      "code": "SM-04",
+      "name": "Faetano",
+      "type": "Municipality"
+    },
+    {
+      "code": "SM-05",
+      "name": "Fiorentino",
+      "type": "Municipality"
+    },
+    {
+      "code": "SM-06",
+      "name": "Borgo Maggiore",
+      "type": "Municipality"
+    },
+    {
+      "code": "SM-07",
+      "name": "Città di San Marino",
+      "type": "Municipality"
+    },
+    {
+      "code": "SM-08",
+      "name": "Montegiardino",
+      "type": "Municipality"
+    },
+    {
+      "code": "SM-09",
+      "name": "Serravalle",
+      "type": "Municipality"
+    },
+    {
+      "code": "SN-DB",
+      "name": "Diourbel",
+      "type": "Region"
+    },
+    {
+      "code": "SN-DK",
+      "name": "Dakar",
+      "type": "Region"
+    },
+    {
+      "code": "SN-FK",
+      "name": "Fatick",
+      "type": "Region"
+    },
+    {
+      "code": "SN-KA",
+      "name": "Kaffrine",
+      "type": "Region"
+    },
+    {
+      "code": "SN-KD",
+      "name": "Kolda",
+      "type": "Region"
+    },
+    {
+      "code": "SN-KE",
+      "name": "Kédougou",
+      "type": "Region"
+    },
+    {
+      "code": "SN-KL",
+      "name": "Kaolack",
+      "type": "Region"
+    },
+    {
+      "code": "SN-LG",
+      "name": "Louga",
+      "type": "Region"
+    },
+    {
+      "code": "SN-MT",
+      "name": "Matam",
+      "type": "Region"
+    },
+    {
+      "code": "SN-SE",
+      "name": "Sédhiou",
+      "type": "Region"
+    },
+    {
+      "code": "SN-SL",
+      "name": "Saint-Louis",
+      "type": "Region"
+    },
+    {
+      "code": "SN-TC",
+      "name": "Tambacounda",
+      "type": "Region"
+    },
+    {
+      "code": "SN-TH",
+      "name": "Thiès",
+      "type": "Region"
+    },
+    {
+      "code": "SN-ZG",
+      "name": "Ziguinchor",
+      "type": "Region"
+    },
+    {
+      "code": "SO-AW",
+      "name": "Awdal",
+      "type": "Region"
+    },
+    {
+      "code": "SO-BK",
+      "name": "Bakool",
+      "type": "Region"
+    },
+    {
+      "code": "SO-BN",
+      "name": "Banaadir",
+      "type": "Region"
+    },
+    {
+      "code": "SO-BR",
+      "name": "Bari",
+      "type": "Region"
+    },
+    {
+      "code": "SO-BY",
+      "name": "Bay",
+      "type": "Region"
+    },
+    {
+      "code": "SO-GA",
+      "name": "Galguduud",
+      "type": "Region"
+    },
+    {
+      "code": "SO-GE",
+      "name": "Gedo",
+      "type": "Region"
+    },
+    {
+      "code": "SO-HI",
+      "name": "Hiiraan",
+      "type": "Region"
+    },
+    {
+      "code": "SO-JD",
+      "name": "Jubbada Dhexe",
+      "type": "Region"
+    },
+    {
+      "code": "SO-JH",
+      "name": "Jubbada Hoose",
+      "type": "Region"
+    },
+    {
+      "code": "SO-MU",
+      "name": "Mudug",
+      "type": "Region"
+    },
+    {
+      "code": "SO-NU",
+      "name": "Nugaal",
+      "type": "Region"
+    },
+    {
+      "code": "SO-SA",
+      "name": "Sanaag",
+      "type": "Region"
+    },
+    {
+      "code": "SO-SD",
+      "name": "Shabeellaha Dhexe",
+      "type": "Region"
+    },
+    {
+      "code": "SO-SH",
+      "name": "Shabeellaha Hoose",
+      "type": "Region"
+    },
+    {
+      "code": "SO-SO",
+      "name": "Sool",
+      "type": "Region"
+    },
+    {
+      "code": "SO-TO",
+      "name": "Togdheer",
+      "type": "Region"
+    },
+    {
+      "code": "SO-WO",
+      "name": "Woqooyi Galbeed",
+      "type": "Region"
+    },
+    {
+      "code": "SR-BR",
+      "name": "Brokopondo",
+      "type": "District"
+    },
+    {
+      "code": "SR-CM",
+      "name": "Commewijne",
+      "type": "District"
+    },
+    {
+      "code": "SR-CR",
+      "name": "Coronie",
+      "type": "District"
+    },
+    {
+      "code": "SR-MA",
+      "name": "Marowijne",
+      "type": "District"
+    },
+    {
+      "code": "SR-NI",
+      "name": "Nickerie",
+      "type": "District"
+    },
+    {
+      "code": "SR-PM",
+      "name": "Paramaribo",
+      "type": "District"
+    },
+    {
+      "code": "SR-PR",
+      "name": "Para",
+      "type": "District"
+    },
+    {
+      "code": "SR-SA",
+      "name": "Saramacca",
+      "type": "District"
+    },
+    {
+      "code": "SR-SI",
+      "name": "Sipaliwini",
+      "type": "District"
+    },
+    {
+      "code": "SR-WA",
+      "name": "Wanica",
+      "type": "District"
+    },
+    {
+      "code": "SS-BN",
+      "name": "Northern Bahr el Ghazal",
+      "type": "State"
+    },
+    {
+      "code": "SS-BW",
+      "name": "Western Bahr el Ghazal",
+      "type": "State"
+    },
+    {
+      "code": "SS-EC",
+      "name": "Central Equatoria",
+      "type": "State"
+    },
+    {
+      "code": "SS-EE",
+      "name": "Eastern Equatoria",
+      "type": "State"
+    },
+    {
+      "code": "SS-EW",
+      "name": "Western Equatoria",
+      "type": "State"
+    },
+    {
+      "code": "SS-JG",
+      "name": "Jonglei",
+      "type": "State"
+    },
+    {
+      "code": "SS-LK",
+      "name": "Lakes",
+      "type": "State"
+    },
+    {
+      "code": "SS-NU",
+      "name": "Upper Nile",
+      "type": "State"
+    },
+    {
+      "code": "SS-UY",
+      "name": "Unity",
+      "type": "State"
+    },
+    {
+      "code": "SS-WR",
+      "name": "Warrap",
+      "type": "State"
+    },
+    {
+      "code": "ST-01",
+      "name": "Água Grande",
+      "type": "District"
+    },
+    {
+      "code": "ST-02",
+      "name": "Cantagalo",
+      "type": "District"
+    },
+    {
+      "code": "ST-03",
+      "name": "Caué",
+      "type": "District"
+    },
+    {
+      "code": "ST-04",
+      "name": "Lembá",
+      "type": "District"
+    },
+    {
+      "code": "ST-05",
+      "name": "Lobata",
+      "type": "District"
+    },
+    {
+      "code": "ST-06",
+      "name": "Mé-Zóchi",
+      "type": "District"
+    },
+    {
+      "code": "ST-P",
+      "name": "Príncipe",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "SV-AH",
+      "name": "Ahuachapán",
+      "type": "Department"
+    },
+    {
+      "code": "SV-CA",
+      "name": "Cabañas",
+      "type": "Department"
+    },
+    {
+      "code": "SV-CH",
+      "name": "Chalatenango",
+      "type": "Department"
+    },
+    {
+      "code": "SV-CU",
+      "name": "Cuscatlán",
+      "type": "Department"
+    },
+    {
+      "code": "SV-LI",
+      "name": "La Libertad",
+      "type": "Department"
+    },
+    {
+      "code": "SV-MO",
+      "name": "Morazán",
+      "type": "Department"
+    },
+    {
+      "code": "SV-PA",
+      "name": "La Paz",
+      "type": "Department"
+    },
+    {
+      "code": "SV-SA",
+      "name": "Santa Ana",
+      "type": "Department"
+    },
+    {
+      "code": "SV-SM",
+      "name": "San Miguel",
+      "type": "Department"
+    },
+    {
+      "code": "SV-SO",
+      "name": "Sonsonate",
+      "type": "Department"
+    },
+    {
+      "code": "SV-SS",
+      "name": "San Salvador",
+      "type": "Department"
+    },
+    {
+      "code": "SV-SV",
+      "name": "San Vicente",
+      "type": "Department"
+    },
+    {
+      "code": "SV-UN",
+      "name": "La Unión",
+      "type": "Department"
+    },
+    {
+      "code": "SV-US",
+      "name": "Usulután",
+      "type": "Department"
+    },
+    {
+      "code": "SY-DI",
+      "name": "Dimashq",
+      "type": "Province"
+    },
+    {
+      "code": "SY-DR",
+      "name": "Dar'ā",
+      "type": "Province"
+    },
+    {
+      "code": "SY-DY",
+      "name": "Dayr az Zawr",
+      "type": "Province"
+    },
+    {
+      "code": "SY-HA",
+      "name": "Al Ḩasakah",
+      "type": "Province"
+    },
+    {
+      "code": "SY-HI",
+      "name": "Ḩimş",
+      "type": "Province"
+    },
+    {
+      "code": "SY-HL",
+      "name": "Ḩalab",
+      "type": "Province"
+    },
+    {
+      "code": "SY-HM",
+      "name": "Ḩamāh",
+      "type": "Province"
+    },
+    {
+      "code": "SY-ID",
+      "name": "Idlib",
+      "type": "Province"
+    },
+    {
+      "code": "SY-LA",
+      "name": "Al Lādhiqīyah",
+      "type": "Province"
+    },
+    {
+      "code": "SY-QU",
+      "name": "Al Qunayţirah",
+      "type": "Province"
+    },
+    {
+      "code": "SY-RA",
+      "name": "Ar Raqqah",
+      "type": "Province"
+    },
+    {
+      "code": "SY-RD",
+      "name": "Rīf Dimashq",
+      "type": "Province"
+    },
+    {
+      "code": "SY-SU",
+      "name": "As Suwaydā'",
+      "type": "Province"
+    },
+    {
+      "code": "SY-TA",
+      "name": "Ţarţūs",
+      "type": "Province"
+    },
+    {
+      "code": "SZ-HH",
+      "name": "Hhohho",
+      "type": "Region"
+    },
+    {
+      "code": "SZ-LU",
+      "name": "Lubombo",
+      "type": "Region"
+    },
+    {
+      "code": "SZ-MA",
+      "name": "Manzini",
+      "type": "Region"
+    },
+    {
+      "code": "SZ-SH",
+      "name": "Shiselweni",
+      "type": "Region"
+    },
+    {
+      "code": "TD-BA",
+      "name": "Batha",
+      "type": "Province"
+    },
+    {
+      "code": "TD-BG",
+      "name": "Bahr el Ghazal",
+      "type": "Province"
+    },
+    {
+      "code": "TD-BO",
+      "name": "Borkou",
+      "type": "Province"
+    },
+    {
+      "code": "TD-CB",
+      "name": "Chari-Baguirmi",
+      "type": "Province"
+    },
+    {
+      "code": "TD-EE",
+      "name": "Ennedi-Est",
+      "type": "Province"
+    },
+    {
+      "code": "TD-EO",
+      "name": "Ennedi-Ouest",
+      "type": "Province"
+    },
+    {
+      "code": "TD-GR",
+      "name": "Guéra",
+      "type": "Province"
+    },
+    {
+      "code": "TD-HL",
+      "name": "Hadjer Lamis",
+      "type": "Province"
+    },
+    {
+      "code": "TD-KA",
+      "name": "Kanem",
+      "type": "Province"
+    },
+    {
+      "code": "TD-LC",
+      "name": "Lac",
+      "type": "Province"
+    },
+    {
+      "code": "TD-LO",
+      "name": "Logone-Occidental",
+      "type": "Province"
+    },
+    {
+      "code": "TD-LR",
+      "name": "Logone-Oriental",
+      "type": "Province"
+    },
+    {
+      "code": "TD-MA",
+      "name": "Mandoul",
+      "type": "Province"
+    },
+    {
+      "code": "TD-MC",
+      "name": "Moyen-Chari",
+      "type": "Province"
+    },
+    {
+      "code": "TD-ME",
+      "name": "Mayo-Kebbi-Est",
+      "type": "Province"
+    },
+    {
+      "code": "TD-MO",
+      "name": "Mayo-Kebbi-Ouest",
+      "type": "Province"
+    },
+    {
+      "code": "TD-ND",
+      "name": "Ville de Ndjamena",
+      "type": "Province"
+    },
+    {
+      "code": "TD-OD",
+      "name": "Ouaddaï",
+      "type": "Province"
+    },
+    {
+      "code": "TD-SA",
+      "name": "Salamat",
+      "type": "Province"
+    },
+    {
+      "code": "TD-SI",
+      "name": "Sila",
+      "type": "Province"
+    },
+    {
+      "code": "TD-TA",
+      "name": "Tandjilé",
+      "type": "Province"
+    },
+    {
+      "code": "TD-TI",
+      "name": "Tibesti",
+      "type": "Province"
+    },
+    {
+      "code": "TD-WF",
+      "name": "Wadi Fira",
+      "type": "Province"
+    },
+    {
+      "code": "TG-C",
+      "name": "Centrale",
+      "type": "Region"
+    },
+    {
+      "code": "TG-K",
+      "name": "Kara",
+      "type": "Region"
+    },
+    {
+      "code": "TG-M",
+      "name": "Maritime (Région)",
+      "type": "Region"
+    },
+    {
+      "code": "TG-P",
+      "name": "Plateaux",
+      "type": "Region"
+    },
+    {
+      "code": "TG-S",
+      "name": "Savanes",
+      "type": "Region"
+    },
+    {
+      "code": "TH-10",
+      "name": "Krung Thep Maha Nakhon",
+      "type": "Metropolitan administration"
+    },
+    {
+      "code": "TH-11",
+      "name": "Samut Prakan",
+      "type": "Province"
+    },
+    {
+      "code": "TH-12",
+      "name": "Nonthaburi",
+      "type": "Province"
+    },
+    {
+      "code": "TH-13",
+      "name": "Pathum Thani",
+      "type": "Province"
+    },
+    {
+      "code": "TH-14",
+      "name": "Phra Nakhon Si Ayutthaya",
+      "type": "Province"
+    },
+    {
+      "code": "TH-15",
+      "name": "Ang Thong",
+      "type": "Province"
+    },
+    {
+      "code": "TH-16",
+      "name": "Lop Buri",
+      "type": "Province"
+    },
+    {
+      "code": "TH-17",
+      "name": "Sing Buri",
+      "type": "Province"
+    },
+    {
+      "code": "TH-18",
+      "name": "Chai Nat",
+      "type": "Province"
+    },
+    {
+      "code": "TH-19",
+      "name": "Saraburi",
+      "type": "Province"
+    },
+    {
+      "code": "TH-20",
+      "name": "Chon Buri",
+      "type": "Province"
+    },
+    {
+      "code": "TH-21",
+      "name": "Rayong",
+      "type": "Province"
+    },
+    {
+      "code": "TH-22",
+      "name": "Chanthaburi",
+      "type": "Province"
+    },
+    {
+      "code": "TH-23",
+      "name": "Trat",
+      "type": "Province"
+    },
+    {
+      "code": "TH-24",
+      "name": "Chachoengsao",
+      "type": "Province"
+    },
+    {
+      "code": "TH-25",
+      "name": "Prachin Buri",
+      "type": "Province"
+    },
+    {
+      "code": "TH-26",
+      "name": "Nakhon Nayok",
+      "type": "Province"
+    },
+    {
+      "code": "TH-27",
+      "name": "Sa Kaeo",
+      "type": "Province"
+    },
+    {
+      "code": "TH-30",
+      "name": "Nakhon Ratchasima",
+      "type": "Province"
+    },
+    {
+      "code": "TH-31",
+      "name": "Buri Ram",
+      "type": "Province"
+    },
+    {
+      "code": "TH-32",
+      "name": "Surin",
+      "type": "Province"
+    },
+    {
+      "code": "TH-33",
+      "name": "Si Sa Ket",
+      "type": "Province"
+    },
+    {
+      "code": "TH-34",
+      "name": "Ubon Ratchathani",
+      "type": "Province"
+    },
+    {
+      "code": "TH-35",
+      "name": "Yasothon",
+      "type": "Province"
+    },
+    {
+      "code": "TH-36",
+      "name": "Chaiyaphum",
+      "type": "Province"
+    },
+    {
+      "code": "TH-37",
+      "name": "Amnat Charoen",
+      "type": "Province"
+    },
+    {
+      "code": "TH-38",
+      "name": "Bueng Kan",
+      "type": "Province"
+    },
+    {
+      "code": "TH-39",
+      "name": "Nong Bua Lam Phu",
+      "type": "Province"
+    },
+    {
+      "code": "TH-40",
+      "name": "Khon Kaen",
+      "type": "Province"
+    },
+    {
+      "code": "TH-41",
+      "name": "Udon Thani",
+      "type": "Province"
+    },
+    {
+      "code": "TH-42",
+      "name": "Loei",
+      "type": "Province"
+    },
+    {
+      "code": "TH-43",
+      "name": "Nong Khai",
+      "type": "Province"
+    },
+    {
+      "code": "TH-44",
+      "name": "Maha Sarakham",
+      "type": "Province"
+    },
+    {
+      "code": "TH-45",
+      "name": "Roi Et",
+      "type": "Province"
+    },
+    {
+      "code": "TH-46",
+      "name": "Kalasin",
+      "type": "Province"
+    },
+    {
+      "code": "TH-47",
+      "name": "Sakon Nakhon",
+      "type": "Province"
+    },
+    {
+      "code": "TH-48",
+      "name": "Nakhon Phanom",
+      "type": "Province"
+    },
+    {
+      "code": "TH-49",
+      "name": "Mukdahan",
+      "type": "Province"
+    },
+    {
+      "code": "TH-50",
+      "name": "Chiang Mai",
+      "type": "Province"
+    },
+    {
+      "code": "TH-51",
+      "name": "Lamphun",
+      "type": "Province"
+    },
+    {
+      "code": "TH-52",
+      "name": "Lampang",
+      "type": "Province"
+    },
+    {
+      "code": "TH-53",
+      "name": "Uttaradit",
+      "type": "Province"
+    },
+    {
+      "code": "TH-54",
+      "name": "Phrae",
+      "type": "Province"
+    },
+    {
+      "code": "TH-55",
+      "name": "Nan",
+      "type": "Province"
+    },
+    {
+      "code": "TH-56",
+      "name": "Phayao",
+      "type": "Province"
+    },
+    {
+      "code": "TH-57",
+      "name": "Chiang Rai",
+      "type": "Province"
+    },
+    {
+      "code": "TH-58",
+      "name": "Mae Hong Son",
+      "type": "Province"
+    },
+    {
+      "code": "TH-60",
+      "name": "Nakhon Sawan",
+      "type": "Province"
+    },
+    {
+      "code": "TH-61",
+      "name": "Uthai Thani",
+      "type": "Province"
+    },
+    {
+      "code": "TH-62",
+      "name": "Kamphaeng Phet",
+      "type": "Province"
+    },
+    {
+      "code": "TH-63",
+      "name": "Tak",
+      "type": "Province"
+    },
+    {
+      "code": "TH-64",
+      "name": "Sukhothai",
+      "type": "Province"
+    },
+    {
+      "code": "TH-65",
+      "name": "Phitsanulok",
+      "type": "Province"
+    },
+    {
+      "code": "TH-66",
+      "name": "Phichit",
+      "type": "Province"
+    },
+    {
+      "code": "TH-67",
+      "name": "Phetchabun",
+      "type": "Province"
+    },
+    {
+      "code": "TH-70",
+      "name": "Ratchaburi",
+      "type": "Province"
+    },
+    {
+      "code": "TH-71",
+      "name": "Kanchanaburi",
+      "type": "Province"
+    },
+    {
+      "code": "TH-72",
+      "name": "Suphan Buri",
+      "type": "Province"
+    },
+    {
+      "code": "TH-73",
+      "name": "Nakhon Pathom",
+      "type": "Province"
+    },
+    {
+      "code": "TH-74",
+      "name": "Samut Sakhon",
+      "type": "Province"
+    },
+    {
+      "code": "TH-75",
+      "name": "Samut Songkhram",
+      "type": "Province"
+    },
+    {
+      "code": "TH-76",
+      "name": "Phetchaburi",
+      "type": "Province"
+    },
+    {
+      "code": "TH-77",
+      "name": "Prachuap Khiri Khan",
+      "type": "Province"
+    },
+    {
+      "code": "TH-80",
+      "name": "Nakhon Si Thammarat",
+      "type": "Province"
+    },
+    {
+      "code": "TH-81",
+      "name": "Krabi",
+      "type": "Province"
+    },
+    {
+      "code": "TH-82",
+      "name": "Phangnga",
+      "type": "Province"
+    },
+    {
+      "code": "TH-83",
+      "name": "Phuket",
+      "type": "Province"
+    },
+    {
+      "code": "TH-84",
+      "name": "Surat Thani",
+      "type": "Province"
+    },
+    {
+      "code": "TH-85",
+      "name": "Ranong",
+      "type": "Province"
+    },
+    {
+      "code": "TH-86",
+      "name": "Chumphon",
+      "type": "Province"
+    },
+    {
+      "code": "TH-90",
+      "name": "Songkhla",
+      "type": "Province"
+    },
+    {
+      "code": "TH-91",
+      "name": "Satun",
+      "type": "Province"
+    },
+    {
+      "code": "TH-92",
+      "name": "Trang",
+      "type": "Province"
+    },
+    {
+      "code": "TH-93",
+      "name": "Phatthalung",
+      "type": "Province"
+    },
+    {
+      "code": "TH-94",
+      "name": "Pattani",
+      "type": "Province"
+    },
+    {
+      "code": "TH-95",
+      "name": "Yala",
+      "type": "Province"
+    },
+    {
+      "code": "TH-96",
+      "name": "Narathiwat",
+      "type": "Province"
+    },
+    {
+      "code": "TH-S",
+      "name": "Phatthaya",
+      "type": "Special administrative city"
+    },
+    {
+      "code": "TJ-DU",
+      "name": "Dushanbe",
+      "type": "Capital territory"
+    },
+    {
+      "code": "TJ-GB",
+      "name": "Kŭhistoni Badakhshon",
+      "type": "Autonomous region"
+    },
+    {
+      "code": "TJ-KT",
+      "name": "Khatlon",
+      "type": "Region"
+    },
+    {
+      "code": "TJ-RA",
+      "name": "nohiyahoi tobei jumhurí",
+      "type": "Districts under republic administration"
+    },
+    {
+      "code": "TJ-SU",
+      "name": "Sughd",
+      "type": "Region"
+    },
+    {
+      "code": "TL-AL",
+      "name": "Aileu",
+      "type": "Municipality"
+    },
+    {
+      "code": "TL-AN",
+      "name": "Ainaro",
+      "type": "Municipality"
+    },
+    {
+      "code": "TL-BA",
+      "name": "Baucau",
+      "type": "Municipality"
+    },
+    {
+      "code": "TL-BO",
+      "name": "Bobonaro",
+      "type": "Municipality"
+    },
+    {
+      "code": "TL-CO",
+      "name": "Cova Lima",
+      "type": "Municipality"
+    },
+    {
+      "code": "TL-DI",
+      "name": "Díli",
+      "type": "Municipality"
+    },
+    {
+      "code": "TL-ER",
+      "name": "Ermera",
+      "type": "Municipality"
+    },
+    {
+      "code": "TL-LA",
+      "name": "Lautein",
+      "type": "Municipality"
+    },
+    {
+      "code": "TL-LI",
+      "name": "Likisá",
+      "type": "Municipality"
+    },
+    {
+      "code": "TL-MF",
+      "name": "Manufahi",
+      "type": "Municipality"
+    },
+    {
+      "code": "TL-MT",
+      "name": "Manatuto",
+      "type": "Municipality"
+    },
+    {
+      "code": "TL-OE",
+      "name": "Oekusi-Ambenu",
+      "type": "Special administrative region"
+    },
+    {
+      "code": "TL-VI",
+      "name": "Vikeke",
+      "type": "Municipality"
+    },
+    {
+      "code": "TM-A",
+      "name": "Ahal",
+      "type": "Region"
+    },
+    {
+      "code": "TM-B",
+      "name": "Balkan",
+      "type": "Region"
+    },
+    {
+      "code": "TM-D",
+      "name": "Daşoguz",
+      "type": "Region"
+    },
+    {
+      "code": "TM-L",
+      "name": "Lebap",
+      "type": "Region"
+    },
+    {
+      "code": "TM-M",
+      "name": "Mary",
+      "type": "Region"
+    },
+    {
+      "code": "TM-S",
+      "name": "Aşgabat",
+      "type": "City"
+    },
+    {
+      "code": "TN-11",
+      "name": "Tunis",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-12",
+      "name": "L'Ariana",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-13",
+      "name": "Ben Arous",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-14",
+      "name": "La Manouba",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-21",
+      "name": "Nabeul",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-22",
+      "name": "Zaghouan",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-23",
+      "name": "Bizerte",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-31",
+      "name": "Béja",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-32",
+      "name": "Jendouba",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-33",
+      "name": "Le Kef",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-34",
+      "name": "Siliana",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-41",
+      "name": "Kairouan",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-42",
+      "name": "Kasserine",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-43",
+      "name": "Sidi Bouzid",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-51",
+      "name": "Sousse",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-52",
+      "name": "Monastir",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-53",
+      "name": "Mahdia",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-61",
+      "name": "Sfax",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-71",
+      "name": "Gafsa",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-72",
+      "name": "Tozeur",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-73",
+      "name": "Kébili",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-81",
+      "name": "Gabès",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-82",
+      "name": "Médenine",
+      "type": "Governorate"
+    },
+    {
+      "code": "TN-83",
+      "name": "Tataouine",
+      "type": "Governorate"
+    },
+    {
+      "code": "TO-01",
+      "name": "'Eua",
+      "type": "Division"
+    },
+    {
+      "code": "TO-02",
+      "name": "Ha'apai",
+      "type": "Division"
+    },
+    {
+      "code": "TO-03",
+      "name": "Niuas",
+      "type": "Division"
+    },
+    {
+      "code": "TO-04",
+      "name": "Tongatapu",
+      "type": "Division"
+    },
+    {
+      "code": "TO-05",
+      "name": "Vava'u",
+      "type": "Division"
+    },
+    {
+      "code": "TR-01",
+      "name": "Adana",
+      "type": "Province"
+    },
+    {
+      "code": "TR-02",
+      "name": "Adıyaman",
+      "type": "Province"
+    },
+    {
+      "code": "TR-03",
+      "name": "Afyonkarahisar",
+      "type": "Province"
+    },
+    {
+      "code": "TR-04",
+      "name": "Ağrı",
+      "type": "Province"
+    },
+    {
+      "code": "TR-05",
+      "name": "Amasya",
+      "type": "Province"
+    },
+    {
+      "code": "TR-06",
+      "name": "Ankara",
+      "type": "Province"
+    },
+    {
+      "code": "TR-07",
+      "name": "Antalya",
+      "type": "Province"
+    },
+    {
+      "code": "TR-08",
+      "name": "Artvin",
+      "type": "Province"
+    },
+    {
+      "code": "TR-09",
+      "name": "Aydın",
+      "type": "Province"
+    },
+    {
+      "code": "TR-10",
+      "name": "Balıkesir",
+      "type": "Province"
+    },
+    {
+      "code": "TR-11",
+      "name": "Bilecik",
+      "type": "Province"
+    },
+    {
+      "code": "TR-12",
+      "name": "Bingöl",
+      "type": "Province"
+    },
+    {
+      "code": "TR-13",
+      "name": "Bitlis",
+      "type": "Province"
+    },
+    {
+      "code": "TR-14",
+      "name": "Bolu",
+      "type": "Province"
+    },
+    {
+      "code": "TR-15",
+      "name": "Burdur",
+      "type": "Province"
+    },
+    {
+      "code": "TR-16",
+      "name": "Bursa",
+      "type": "Province"
+    },
+    {
+      "code": "TR-17",
+      "name": "Çanakkale",
+      "type": "Province"
+    },
+    {
+      "code": "TR-18",
+      "name": "Çankırı",
+      "type": "Province"
+    },
+    {
+      "code": "TR-19",
+      "name": "Çorum",
+      "type": "Province"
+    },
+    {
+      "code": "TR-20",
+      "name": "Denizli",
+      "type": "Province"
+    },
+    {
+      "code": "TR-21",
+      "name": "Diyarbakır",
+      "type": "Province"
+    },
+    {
+      "code": "TR-22",
+      "name": "Edirne",
+      "type": "Province"
+    },
+    {
+      "code": "TR-23",
+      "name": "Elazığ",
+      "type": "Province"
+    },
+    {
+      "code": "TR-24",
+      "name": "Erzincan",
+      "type": "Province"
+    },
+    {
+      "code": "TR-25",
+      "name": "Erzurum",
+      "type": "Province"
+    },
+    {
+      "code": "TR-26",
+      "name": "Eskişehir",
+      "type": "Province"
+    },
+    {
+      "code": "TR-27",
+      "name": "Gaziantep",
+      "type": "Province"
+    },
+    {
+      "code": "TR-28",
+      "name": "Giresun",
+      "type": "Province"
+    },
+    {
+      "code": "TR-29",
+      "name": "Gümüşhane",
+      "type": "Province"
+    },
+    {
+      "code": "TR-30",
+      "name": "Hakkâri",
+      "type": "Province"
+    },
+    {
+      "code": "TR-31",
+      "name": "Hatay",
+      "type": "Province"
+    },
+    {
+      "code": "TR-32",
+      "name": "Isparta",
+      "type": "Province"
+    },
+    {
+      "code": "TR-33",
+      "name": "Mersin",
+      "type": "Province"
+    },
+    {
+      "code": "TR-34",
+      "name": "İstanbul",
+      "type": "Province"
+    },
+    {
+      "code": "TR-35",
+      "name": "İzmir",
+      "type": "Province"
+    },
+    {
+      "code": "TR-36",
+      "name": "Kars",
+      "type": "Province"
+    },
+    {
+      "code": "TR-37",
+      "name": "Kastamonu",
+      "type": "Province"
+    },
+    {
+      "code": "TR-38",
+      "name": "Kayseri",
+      "type": "Province"
+    },
+    {
+      "code": "TR-39",
+      "name": "Kırklareli",
+      "type": "Province"
+    },
+    {
+      "code": "TR-40",
+      "name": "Kırşehir",
+      "type": "Province"
+    },
+    {
+      "code": "TR-41",
+      "name": "Kocaeli",
+      "type": "Province"
+    },
+    {
+      "code": "TR-42",
+      "name": "Konya",
+      "type": "Province"
+    },
+    {
+      "code": "TR-43",
+      "name": "Kütahya",
+      "type": "Province"
+    },
+    {
+      "code": "TR-44",
+      "name": "Malatya",
+      "type": "Province"
+    },
+    {
+      "code": "TR-45",
+      "name": "Manisa",
+      "type": "Province"
+    },
+    {
+      "code": "TR-46",
+      "name": "Kahramanmaraş",
+      "type": "Province"
+    },
+    {
+      "code": "TR-47",
+      "name": "Mardin",
+      "type": "Province"
+    },
+    {
+      "code": "TR-48",
+      "name": "Muğla",
+      "type": "Province"
+    },
+    {
+      "code": "TR-49",
+      "name": "Muş",
+      "type": "Province"
+    },
+    {
+      "code": "TR-50",
+      "name": "Nevşehir",
+      "type": "Province"
+    },
+    {
+      "code": "TR-51",
+      "name": "Niğde",
+      "type": "Province"
+    },
+    {
+      "code": "TR-52",
+      "name": "Ordu",
+      "type": "Province"
+    },
+    {
+      "code": "TR-53",
+      "name": "Rize",
+      "type": "Province"
+    },
+    {
+      "code": "TR-54",
+      "name": "Sakarya",
+      "type": "Province"
+    },
+    {
+      "code": "TR-55",
+      "name": "Samsun",
+      "type": "Province"
+    },
+    {
+      "code": "TR-56",
+      "name": "Siirt",
+      "type": "Province"
+    },
+    {
+      "code": "TR-57",
+      "name": "Sinop",
+      "type": "Province"
+    },
+    {
+      "code": "TR-58",
+      "name": "Sivas",
+      "type": "Province"
+    },
+    {
+      "code": "TR-59",
+      "name": "Tekirdağ",
+      "type": "Province"
+    },
+    {
+      "code": "TR-60",
+      "name": "Tokat",
+      "type": "Province"
+    },
+    {
+      "code": "TR-61",
+      "name": "Trabzon",
+      "type": "Province"
+    },
+    {
+      "code": "TR-62",
+      "name": "Tunceli",
+      "type": "Province"
+    },
+    {
+      "code": "TR-63",
+      "name": "Şanlıurfa",
+      "type": "Province"
+    },
+    {
+      "code": "TR-64",
+      "name": "Uşak",
+      "type": "Province"
+    },
+    {
+      "code": "TR-65",
+      "name": "Van",
+      "type": "Province"
+    },
+    {
+      "code": "TR-66",
+      "name": "Yozgat",
+      "type": "Province"
+    },
+    {
+      "code": "TR-67",
+      "name": "Zonguldak",
+      "type": "Province"
+    },
+    {
+      "code": "TR-68",
+      "name": "Aksaray",
+      "type": "Province"
+    },
+    {
+      "code": "TR-69",
+      "name": "Bayburt",
+      "type": "Province"
+    },
+    {
+      "code": "TR-70",
+      "name": "Karaman",
+      "type": "Province"
+    },
+    {
+      "code": "TR-71",
+      "name": "Kırıkkale",
+      "type": "Province"
+    },
+    {
+      "code": "TR-72",
+      "name": "Batman",
+      "type": "Province"
+    },
+    {
+      "code": "TR-73",
+      "name": "Şırnak",
+      "type": "Province"
+    },
+    {
+      "code": "TR-74",
+      "name": "Bartın",
+      "type": "Province"
+    },
+    {
+      "code": "TR-75",
+      "name": "Ardahan",
+      "type": "Province"
+    },
+    {
+      "code": "TR-76",
+      "name": "Iğdır",
+      "type": "Province"
+    },
+    {
+      "code": "TR-77",
+      "name": "Yalova",
+      "type": "Province"
+    },
+    {
+      "code": "TR-78",
+      "name": "Karabük",
+      "type": "Province"
+    },
+    {
+      "code": "TR-79",
+      "name": "Kilis",
+      "type": "Province"
+    },
+    {
+      "code": "TR-80",
+      "name": "Osmaniye",
+      "type": "Province"
+    },
+    {
+      "code": "TR-81",
+      "name": "Düzce",
+      "type": "Province"
+    },
+    {
+      "code": "TT-ARI",
+      "name": "Arima",
+      "type": "Borough"
+    },
+    {
+      "code": "TT-CHA",
+      "name": "Chaguanas",
+      "type": "Borough"
+    },
+    {
+      "code": "TT-CTT",
+      "name": "Couva-Tabaquite-Talparo",
+      "type": "Region"
+    },
+    {
+      "code": "TT-DMN",
+      "name": "Diego Martin",
+      "type": "Region"
+    },
+    {
+      "code": "TT-MRC",
+      "name": "Mayaro-Rio Claro",
+      "type": "Region"
+    },
+    {
+      "code": "TT-PED",
+      "name": "Penal-Debe",
+      "type": "Region"
+    },
+    {
+      "code": "TT-POS",
+      "name": "Port of Spain",
+      "type": "City"
+    },
+    {
+      "code": "TT-PRT",
+      "name": "Princes Town",
+      "type": "Region"
+    },
+    {
+      "code": "TT-PTF",
+      "name": "Point Fortin",
+      "type": "Borough"
+    },
+    {
+      "code": "TT-SFO",
+      "name": "San Fernando",
+      "type": "City"
+    },
+    {
+      "code": "TT-SGE",
+      "name": "Sangre Grande",
+      "type": "Region"
+    },
+    {
+      "code": "TT-SIP",
+      "name": "Siparia",
+      "type": "Region"
+    },
+    {
+      "code": "TT-SJL",
+      "name": "San Juan-Laventille",
+      "type": "Region"
+    },
+    {
+      "code": "TT-TOB",
+      "name": "Tobago",
+      "type": "Ward"
+    },
+    {
+      "code": "TT-TUP",
+      "name": "Tunapuna-Piarco",
+      "type": "Region"
+    },
+    {
+      "code": "TV-FUN",
+      "name": "Funafuti",
+      "type": "Town council"
+    },
+    {
+      "code": "TV-NIT",
+      "name": "Niutao",
+      "type": "Island council"
+    },
+    {
+      "code": "TV-NKF",
+      "name": "Nukufetau",
+      "type": "Island council"
+    },
+    {
+      "code": "TV-NKL",
+      "name": "Nukulaelae",
+      "type": "Island council"
+    },
+    {
+      "code": "TV-NMA",
+      "name": "Nanumea",
+      "type": "Island council"
+    },
+    {
+      "code": "TV-NMG",
+      "name": "Nanumaga",
+      "type": "Island council"
+    },
+    {
+      "code": "TV-NUI",
+      "name": "Nui",
+      "type": "Island council"
+    },
+    {
+      "code": "TV-VAI",
+      "name": "Vaitupu",
+      "type": "Island council"
+    },
+    {
+      "code": "TW-CHA",
+      "name": "Changhua",
+      "type": "County"
+    },
+    {
+      "code": "TW-CYI",
+      "name": "Chiayi",
+      "type": "City"
+    },
+    {
+      "code": "TW-CYQ",
+      "name": "Chiayi",
+      "type": "County"
+    },
+    {
+      "code": "TW-HSQ",
+      "name": "Hsinchu",
+      "type": "County"
+    },
+    {
+      "code": "TW-HSZ",
+      "name": "Hsinchu",
+      "type": "City"
+    },
+    {
+      "code": "TW-HUA",
+      "name": "Hualien",
+      "type": "County"
+    },
+    {
+      "code": "TW-ILA",
+      "name": "Yilan",
+      "type": "County"
+    },
+    {
+      "code": "TW-KEE",
+      "name": "Keelung",
+      "type": "City"
+    },
+    {
+      "code": "TW-KHH",
+      "name": "Kaohsiung",
+      "type": "Special municipality"
+    },
+    {
+      "code": "TW-KIN",
+      "name": "Kinmen",
+      "type": "County"
+    },
+    {
+      "code": "TW-LIE",
+      "name": "Lienchiang",
+      "type": "County"
+    },
+    {
+      "code": "TW-MIA",
+      "name": "Miaoli",
+      "type": "County"
+    },
+    {
+      "code": "TW-NAN",
+      "name": "Nantou",
+      "type": "County"
+    },
+    {
+      "code": "TW-NWT",
+      "name": "New Taipei",
+      "type": "Special municipality"
+    },
+    {
+      "code": "TW-PEN",
+      "name": "Penghu",
+      "type": "County"
+    },
+    {
+      "code": "TW-PIF",
+      "name": "Pingtung",
+      "type": "County"
+    },
+    {
+      "code": "TW-TAO",
+      "name": "Taoyuan",
+      "type": "Special municipality"
+    },
+    {
+      "code": "TW-TNN",
+      "name": "Tainan",
+      "type": "Special municipality"
+    },
+    {
+      "code": "TW-TPE",
+      "name": "Taipei",
+      "type": "Special municipality"
+    },
+    {
+      "code": "TW-TTT",
+      "name": "Taitung",
+      "type": "County"
+    },
+    {
+      "code": "TW-TXG",
+      "name": "Taichung",
+      "type": "Special municipality"
+    },
+    {
+      "code": "TW-YUN",
+      "name": "Yunlin",
+      "type": "County"
+    },
+    {
+      "code": "TZ-01",
+      "name": "Arusha",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-02",
+      "name": "Dar es Salaam",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-03",
+      "name": "Dodoma",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-04",
+      "name": "Iringa",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-05",
+      "name": "Kagera",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-06",
+      "name": "Pemba North",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-07",
+      "name": "Zanzibar North",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-08",
+      "name": "Kigoma",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-09",
+      "name": "Kilimanjaro",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-10",
+      "name": "Pemba South",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-11",
+      "name": "Zanzibar South",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-12",
+      "name": "Lindi",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-13",
+      "name": "Mara",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-14",
+      "name": "Mbeya",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-15",
+      "name": "Zanzibar West",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-16",
+      "name": "Morogoro",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-17",
+      "name": "Mtwara",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-18",
+      "name": "Mwanza",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-19",
+      "name": "Coast",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-20",
+      "name": "Rukwa",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-21",
+      "name": "Ruvuma",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-22",
+      "name": "Shinyanga",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-23",
+      "name": "Singida",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-24",
+      "name": "Tabora",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-25",
+      "name": "Tanga",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-26",
+      "name": "Manyara",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-27",
+      "name": "Geita",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-28",
+      "name": "Katavi",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-29",
+      "name": "Njombe",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-30",
+      "name": "Simiyu",
+      "type": "Region"
+    },
+    {
+      "code": "TZ-31",
+      "name": "Songwe",
+      "type": "Region"
+    },
+    {
+      "code": "UA-05",
+      "name": "Vinnytska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-07",
+      "name": "Volynska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-09",
+      "name": "Luhanska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-12",
+      "name": "Dnipropetrovska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-14",
+      "name": "Donetska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-18",
+      "name": "Zhytomyrska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-21",
+      "name": "Zakarpatska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-23",
+      "name": "Zaporizka oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-26",
+      "name": "Ivano-Frankivska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-30",
+      "name": "Kyiv",
+      "type": "City"
+    },
+    {
+      "code": "UA-32",
+      "name": "Kyivska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-35",
+      "name": "Kirovohradska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-40",
+      "name": "Sevastopol",
+      "type": "City"
+    },
+    {
+      "code": "UA-43",
+      "name": "Avtonomna Respublika Krym",
+      "type": "Republic"
+    },
+    {
+      "code": "UA-46",
+      "name": "Lvivska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-48",
+      "name": "Mykolaivska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-51",
+      "name": "Odeska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-53",
+      "name": "Poltavska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-56",
+      "name": "Rivnenska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-59",
+      "name": "Sumska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-61",
+      "name": "Ternopilska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-63",
+      "name": "Kharkivska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-65",
+      "name": "Khersonska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-68",
+      "name": "Khmelnytska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-71",
+      "name": "Cherkaska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-74",
+      "name": "Chernihivska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UA-77",
+      "name": "Chernivetska oblast",
+      "type": "Region"
+    },
+    {
+      "code": "UG-101",
+      "name": "Kalangala",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-102",
+      "name": "Kampala",
+      "parent": "UG-C",
+      "type": "City"
+    },
+    {
+      "code": "UG-103",
+      "name": "Kiboga",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-104",
+      "name": "Luwero",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-105",
+      "name": "Masaka",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-106",
+      "name": "Mpigi",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-107",
+      "name": "Mubende",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-108",
+      "name": "Mukono",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-109",
+      "name": "Nakasongola",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-110",
+      "name": "Rakai",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-111",
+      "name": "Sembabule",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-112",
+      "name": "Kayunga",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-113",
+      "name": "Wakiso",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-114",
+      "name": "Lyantonde",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-115",
+      "name": "Mityana",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-116",
+      "name": "Nakaseke",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-117",
+      "name": "Buikwe",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-118",
+      "name": "Bukomansibi",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-119",
+      "name": "Butambala",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-120",
+      "name": "Buvuma",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-121",
+      "name": "Gomba",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-122",
+      "name": "Kalungu",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-123",
+      "name": "Kyankwanzi",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-124",
+      "name": "Lwengo",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-125",
+      "name": "Kyotera",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-126",
+      "name": "Kasanda",
+      "parent": "UG-C",
+      "type": "District"
+    },
+    {
+      "code": "UG-201",
+      "name": "Bugiri",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-202",
+      "name": "Busia",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-203",
+      "name": "Iganga",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-204",
+      "name": "Jinja",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-205",
+      "name": "Kamuli",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-206",
+      "name": "Kapchorwa",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-207",
+      "name": "Katakwi",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-208",
+      "name": "Kumi",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-209",
+      "name": "Mbale",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-210",
+      "name": "Pallisa",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-211",
+      "name": "Soroti",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-212",
+      "name": "Tororo",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-213",
+      "name": "Kaberamaido",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-214",
+      "name": "Mayuge",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-215",
+      "name": "Sironko",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-216",
+      "name": "Amuria",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-217",
+      "name": "Budaka",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-218",
+      "name": "Bududa",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-219",
+      "name": "Bukedea",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-220",
+      "name": "Bukwo",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-221",
+      "name": "Butaleja",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-222",
+      "name": "Kaliro",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-223",
+      "name": "Manafwa",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-224",
+      "name": "Namutumba",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-225",
+      "name": "Bulambuli",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-226",
+      "name": "Buyende",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-227",
+      "name": "Kibuku",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-228",
+      "name": "Kween",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-229",
+      "name": "Luuka",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-230",
+      "name": "Namayingo",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-231",
+      "name": "Ngora",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-232",
+      "name": "Serere",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-233",
+      "name": "Butebo",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-234",
+      "name": "Namisindwa",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-235",
+      "name": "Bugweri",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-236",
+      "name": "Kapelebyong",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-237",
+      "name": "Kalaki",
+      "parent": "UG-E",
+      "type": "District"
+    },
+    {
+      "code": "UG-301",
+      "name": "Adjumani",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-302",
+      "name": "Apac",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-303",
+      "name": "Arua",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-304",
+      "name": "Gulu",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-305",
+      "name": "Kitgum",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-306",
+      "name": "Kotido",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-307",
+      "name": "Lira",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-308",
+      "name": "Moroto",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-309",
+      "name": "Moyo",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-310",
+      "name": "Nebbi",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-311",
+      "name": "Nakapiripirit",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-312",
+      "name": "Pader",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-313",
+      "name": "Yumbe",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-314",
+      "name": "Abim",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-315",
+      "name": "Amolatar",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-316",
+      "name": "Amuru",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-317",
+      "name": "Dokolo",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-318",
+      "name": "Kaabong",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-319",
+      "name": "Koboko",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-320",
+      "name": "Maracha",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-321",
+      "name": "Oyam",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-322",
+      "name": "Agago",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-323",
+      "name": "Alebtong",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-324",
+      "name": "Amudat",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-325",
+      "name": "Kole",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-326",
+      "name": "Lamwo",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-327",
+      "name": "Napak",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-328",
+      "name": "Nwoya",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-329",
+      "name": "Otuke",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-330",
+      "name": "Zombo",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-331",
+      "name": "Omoro",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-332",
+      "name": "Pakwach",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-333",
+      "name": "Kwania",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-334",
+      "name": "Nabilatuk",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-335",
+      "name": "Karenga",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-336",
+      "name": "Madi-Okollo",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-337",
+      "name": "Obongi",
+      "parent": "UG-N",
+      "type": "District"
+    },
+    {
+      "code": "UG-401",
+      "name": "Bundibugyo",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-402",
+      "name": "Bushenyi",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-403",
+      "name": "Hoima",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-404",
+      "name": "Kabale",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-405",
+      "name": "Kabarole",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-406",
+      "name": "Kasese",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-407",
+      "name": "Kibaale",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-408",
+      "name": "Kisoro",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-409",
+      "name": "Masindi",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-410",
+      "name": "Mbarara",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-411",
+      "name": "Ntungamo",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-412",
+      "name": "Rukungiri",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-413",
+      "name": "Kamwenge",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-414",
+      "name": "Kanungu",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-415",
+      "name": "Kyenjojo",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-416",
+      "name": "Buliisa",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-417",
+      "name": "Ibanda",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-418",
+      "name": "Isingiro",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-419",
+      "name": "Kiruhura",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-420",
+      "name": "Buhweju",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-421",
+      "name": "Kiryandongo",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-422",
+      "name": "Kyegegwa",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-423",
+      "name": "Mitooma",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-424",
+      "name": "Ntoroko",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-425",
+      "name": "Rubirizi",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-426",
+      "name": "Sheema",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-427",
+      "name": "Kagadi",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-428",
+      "name": "Kakumiro",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-429",
+      "name": "Rubanda",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-430",
+      "name": "Bunyangabu",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-431",
+      "name": "Rukiga",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-432",
+      "name": "Kikuube",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-433",
+      "name": "Kazo",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-434",
+      "name": "Kitagwenda",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-435",
+      "name": "Rwampara",
+      "parent": "UG-W",
+      "type": "District"
+    },
+    {
+      "code": "UG-C",
+      "name": "Central",
+      "type": "Geographical region"
+    },
+    {
+      "code": "UG-E",
+      "name": "Eastern",
+      "type": "Geographical region"
+    },
+    {
+      "code": "UG-N",
+      "name": "Northern",
+      "type": "Geographical region"
+    },
+    {
+      "code": "UG-W",
+      "name": "Western",
+      "type": "Geographical region"
+    },
+    {
+      "code": "UM-67",
+      "name": "Johnston Atoll",
+      "type": "Islands, groups of islands"
+    },
+    {
+      "code": "UM-71",
+      "name": "Midway Islands",
+      "type": "Islands, groups of islands"
+    },
+    {
+      "code": "UM-76",
+      "name": "Navassa Island",
+      "type": "Islands, groups of islands"
+    },
+    {
+      "code": "UM-79",
+      "name": "Wake Island",
+      "type": "Islands, groups of islands"
+    },
+    {
+      "code": "UM-81",
+      "name": "Baker Island",
+      "type": "Islands, groups of islands"
+    },
+    {
+      "code": "UM-84",
+      "name": "Howland Island",
+      "type": "Islands, groups of islands"
+    },
+    {
+      "code": "UM-86",
+      "name": "Jarvis Island",
+      "type": "Islands, groups of islands"
+    },
+    {
+      "code": "UM-89",
+      "name": "Kingman Reef",
+      "type": "Islands, groups of islands"
+    },
+    {
+      "code": "UM-95",
+      "name": "Palmyra Atoll",
+      "type": "Islands, groups of islands"
+    },
+    {
+      "code": "US-AK",
+      "name": "Alaska",
+      "type": "State"
+    },
+    {
+      "code": "US-AL",
+      "name": "Alabama",
+      "type": "State"
+    },
+    {
+      "code": "US-AR",
+      "name": "Arkansas",
+      "type": "State"
+    },
+    {
+      "code": "US-AS",
+      "name": "American Samoa",
+      "type": "Outlying area"
+    },
+    {
+      "code": "US-AZ",
+      "name": "Arizona",
+      "type": "State"
+    },
+    {
+      "code": "US-CA",
+      "name": "California",
+      "type": "State"
+    },
+    {
+      "code": "US-CO",
+      "name": "Colorado",
+      "type": "State"
+    },
+    {
+      "code": "US-CT",
+      "name": "Connecticut",
+      "type": "State"
+    },
+    {
+      "code": "US-DC",
+      "name": "District of Columbia",
+      "type": "District"
+    },
+    {
+      "code": "US-DE",
+      "name": "Delaware",
+      "type": "State"
+    },
+    {
+      "code": "US-FL",
+      "name": "Florida",
+      "type": "State"
+    },
+    {
+      "code": "US-GA",
+      "name": "Georgia",
+      "type": "State"
+    },
+    {
+      "code": "US-GU",
+      "name": "Guam",
+      "type": "Outlying area"
+    },
+    {
+      "code": "US-HI",
+      "name": "Hawaii",
+      "type": "State"
+    },
+    {
+      "code": "US-IA",
+      "name": "Iowa",
+      "type": "State"
+    },
+    {
+      "code": "US-ID",
+      "name": "Idaho",
+      "type": "State"
+    },
+    {
+      "code": "US-IL",
+      "name": "Illinois",
+      "type": "State"
+    },
+    {
+      "code": "US-IN",
+      "name": "Indiana",
+      "type": "State"
+    },
+    {
+      "code": "US-KS",
+      "name": "Kansas",
+      "type": "State"
+    },
+    {
+      "code": "US-KY",
+      "name": "Kentucky",
+      "type": "State"
+    },
+    {
+      "code": "US-LA",
+      "name": "Louisiana",
+      "type": "State"
+    },
+    {
+      "code": "US-MA",
+      "name": "Massachusetts",
+      "type": "State"
+    },
+    {
+      "code": "US-MD",
+      "name": "Maryland",
+      "type": "State"
+    },
+    {
+      "code": "US-ME",
+      "name": "Maine",
+      "type": "State"
+    },
+    {
+      "code": "US-MI",
+      "name": "Michigan",
+      "type": "State"
+    },
+    {
+      "code": "US-MN",
+      "name": "Minnesota",
+      "type": "State"
+    },
+    {
+      "code": "US-MO",
+      "name": "Missouri",
+      "type": "State"
+    },
+    {
+      "code": "US-MP",
+      "name": "Northern Mariana Islands",
+      "type": "Outlying area"
+    },
+    {
+      "code": "US-MS",
+      "name": "Mississippi",
+      "type": "State"
+    },
+    {
+      "code": "US-MT",
+      "name": "Montana",
+      "type": "State"
+    },
+    {
+      "code": "US-NC",
+      "name": "North Carolina",
+      "type": "State"
+    },
+    {
+      "code": "US-ND",
+      "name": "North Dakota",
+      "type": "State"
+    },
+    {
+      "code": "US-NE",
+      "name": "Nebraska",
+      "type": "State"
+    },
+    {
+      "code": "US-NH",
+      "name": "New Hampshire",
+      "type": "State"
+    },
+    {
+      "code": "US-NJ",
+      "name": "New Jersey",
+      "type": "State"
+    },
+    {
+      "code": "US-NM",
+      "name": "New Mexico",
+      "type": "State"
+    },
+    {
+      "code": "US-NV",
+      "name": "Nevada",
+      "type": "State"
+    },
+    {
+      "code": "US-NY",
+      "name": "New York",
+      "type": "State"
+    },
+    {
+      "code": "US-OH",
+      "name": "Ohio",
+      "type": "State"
+    },
+    {
+      "code": "US-OK",
+      "name": "Oklahoma",
+      "type": "State"
+    },
+    {
+      "code": "US-OR",
+      "name": "Oregon",
+      "type": "State"
+    },
+    {
+      "code": "US-PA",
+      "name": "Pennsylvania",
+      "type": "State"
+    },
+    {
+      "code": "US-PR",
+      "name": "Puerto Rico",
+      "type": "Outlying area"
+    },
+    {
+      "code": "US-RI",
+      "name": "Rhode Island",
+      "type": "State"
+    },
+    {
+      "code": "US-SC",
+      "name": "South Carolina",
+      "type": "State"
+    },
+    {
+      "code": "US-SD",
+      "name": "South Dakota",
+      "type": "State"
+    },
+    {
+      "code": "US-TN",
+      "name": "Tennessee",
+      "type": "State"
+    },
+    {
+      "code": "US-TX",
+      "name": "Texas",
+      "type": "State"
+    },
+    {
+      "code": "US-UM",
+      "name": "United States Minor Outlying Islands",
+      "type": "Outlying area"
+    },
+    {
+      "code": "US-UT",
+      "name": "Utah",
+      "type": "State"
+    },
+    {
+      "code": "US-VA",
+      "name": "Virginia",
+      "type": "State"
+    },
+    {
+      "code": "US-VI",
+      "name": "Virgin Islands, U.S.",
+      "type": "Outlying area"
+    },
+    {
+      "code": "US-VT",
+      "name": "Vermont",
+      "type": "State"
+    },
+    {
+      "code": "US-WA",
+      "name": "Washington",
+      "type": "State"
+    },
+    {
+      "code": "US-WI",
+      "name": "Wisconsin",
+      "type": "State"
+    },
+    {
+      "code": "US-WV",
+      "name": "West Virginia",
+      "type": "State"
+    },
+    {
+      "code": "US-WY",
+      "name": "Wyoming",
+      "type": "State"
+    },
+    {
+      "code": "UY-AR",
+      "name": "Artigas",
+      "type": "Department"
+    },
+    {
+      "code": "UY-CA",
+      "name": "Canelones",
+      "type": "Department"
+    },
+    {
+      "code": "UY-CL",
+      "name": "Cerro Largo",
+      "type": "Department"
+    },
+    {
+      "code": "UY-CO",
+      "name": "Colonia",
+      "type": "Department"
+    },
+    {
+      "code": "UY-DU",
+      "name": "Durazno",
+      "type": "Department"
+    },
+    {
+      "code": "UY-FD",
+      "name": "Florida",
+      "type": "Department"
+    },
+    {
+      "code": "UY-FS",
+      "name": "Flores",
+      "type": "Department"
+    },
+    {
+      "code": "UY-LA",
+      "name": "Lavalleja",
+      "type": "Department"
+    },
+    {
+      "code": "UY-MA",
+      "name": "Maldonado",
+      "type": "Department"
+    },
+    {
+      "code": "UY-MO",
+      "name": "Montevideo",
+      "type": "Department"
+    },
+    {
+      "code": "UY-PA",
+      "name": "Paysandú",
+      "type": "Department"
+    },
+    {
+      "code": "UY-RN",
+      "name": "Río Negro",
+      "type": "Department"
+    },
+    {
+      "code": "UY-RO",
+      "name": "Rocha",
+      "type": "Department"
+    },
+    {
+      "code": "UY-RV",
+      "name": "Rivera",
+      "type": "Department"
+    },
+    {
+      "code": "UY-SA",
+      "name": "Salto",
+      "type": "Department"
+    },
+    {
+      "code": "UY-SJ",
+      "name": "San José",
+      "type": "Department"
+    },
+    {
+      "code": "UY-SO",
+      "name": "Soriano",
+      "type": "Department"
+    },
+    {
+      "code": "UY-TA",
+      "name": "Tacuarembó",
+      "type": "Department"
+    },
+    {
+      "code": "UY-TT",
+      "name": "Treinta y Tres",
+      "type": "Department"
+    },
+    {
+      "code": "UZ-AN",
+      "name": "Andijon",
+      "type": "Region"
+    },
+    {
+      "code": "UZ-BU",
+      "name": "Buxoro",
+      "type": "Region"
+    },
+    {
+      "code": "UZ-FA",
+      "name": "Farg‘ona",
+      "type": "Region"
+    },
+    {
+      "code": "UZ-JI",
+      "name": "Jizzax",
+      "type": "Region"
+    },
+    {
+      "code": "UZ-NG",
+      "name": "Namangan",
+      "type": "Region"
+    },
+    {
+      "code": "UZ-NW",
+      "name": "Navoiy",
+      "type": "Region"
+    },
+    {
+      "code": "UZ-QA",
+      "name": "Qashqadaryo",
+      "type": "Region"
+    },
+    {
+      "code": "UZ-QR",
+      "name": "Qoraqalpog‘iston Respublikasi",
+      "type": "Republic"
+    },
+    {
+      "code": "UZ-SA",
+      "name": "Samarqand",
+      "type": "Region"
+    },
+    {
+      "code": "UZ-SI",
+      "name": "Sirdaryo",
+      "type": "Region"
+    },
+    {
+      "code": "UZ-SU",
+      "name": "Surxondaryo",
+      "type": "Region"
+    },
+    {
+      "code": "UZ-TK",
+      "name": "Toshkent",
+      "type": "City"
+    },
+    {
+      "code": "UZ-TO",
+      "name": "Toshkent",
+      "type": "Region"
+    },
+    {
+      "code": "UZ-XO",
+      "name": "Xorazm",
+      "type": "Region"
+    },
+    {
+      "code": "VC-01",
+      "name": "Charlotte",
+      "type": "Parish"
+    },
+    {
+      "code": "VC-02",
+      "name": "Saint Andrew",
+      "type": "Parish"
+    },
+    {
+      "code": "VC-03",
+      "name": "Saint David",
+      "type": "Parish"
+    },
+    {
+      "code": "VC-04",
+      "name": "Saint George",
+      "type": "Parish"
+    },
+    {
+      "code": "VC-05",
+      "name": "Saint Patrick",
+      "type": "Parish"
+    },
+    {
+      "code": "VC-06",
+      "name": "Grenadines",
+      "type": "Parish"
+    },
+    {
+      "code": "VE-A",
+      "name": "Distrito Capital",
+      "type": "Capital district"
+    },
+    {
+      "code": "VE-B",
+      "name": "Anzoátegui",
+      "type": "State"
+    },
+    {
+      "code": "VE-C",
+      "name": "Apure",
+      "type": "State"
+    },
+    {
+      "code": "VE-D",
+      "name": "Aragua",
+      "type": "State"
+    },
+    {
+      "code": "VE-E",
+      "name": "Barinas",
+      "type": "State"
+    },
+    {
+      "code": "VE-F",
+      "name": "Bolívar",
+      "type": "State"
+    },
+    {
+      "code": "VE-G",
+      "name": "Carabobo",
+      "type": "State"
+    },
+    {
+      "code": "VE-H",
+      "name": "Cojedes",
+      "type": "State"
+    },
+    {
+      "code": "VE-I",
+      "name": "Falcón",
+      "type": "State"
+    },
+    {
+      "code": "VE-J",
+      "name": "Guárico",
+      "type": "State"
+    },
+    {
+      "code": "VE-K",
+      "name": "Lara",
+      "type": "State"
+    },
+    {
+      "code": "VE-L",
+      "name": "Mérida",
+      "type": "State"
+    },
+    {
+      "code": "VE-M",
+      "name": "Miranda",
+      "type": "State"
+    },
+    {
+      "code": "VE-N",
+      "name": "Monagas",
+      "type": "State"
+    },
+    {
+      "code": "VE-O",
+      "name": "Nueva Esparta",
+      "type": "State"
+    },
+    {
+      "code": "VE-P",
+      "name": "Portuguesa",
+      "type": "State"
+    },
+    {
+      "code": "VE-R",
+      "name": "Sucre",
+      "type": "State"
+    },
+    {
+      "code": "VE-S",
+      "name": "Táchira",
+      "type": "State"
+    },
+    {
+      "code": "VE-T",
+      "name": "Trujillo",
+      "type": "State"
+    },
+    {
+      "code": "VE-U",
+      "name": "Yaracuy",
+      "type": "State"
+    },
+    {
+      "code": "VE-V",
+      "name": "Zulia",
+      "type": "State"
+    },
+    {
+      "code": "VE-W",
+      "name": "Dependencias Federales",
+      "type": "Federal dependency"
+    },
+    {
+      "code": "VE-X",
+      "name": "La Guaira",
+      "type": "State"
+    },
+    {
+      "code": "VE-Y",
+      "name": "Delta Amacuro",
+      "type": "State"
+    },
+    {
+      "code": "VE-Z",
+      "name": "Amazonas",
+      "type": "State"
+    },
+    {
+      "code": "VN-01",
+      "name": "Lai Châu",
+      "type": "Province"
+    },
+    {
+      "code": "VN-02",
+      "name": "Lào Cai",
+      "type": "Province"
+    },
+    {
+      "code": "VN-03",
+      "name": "Hà Giang",
+      "type": "Province"
+    },
+    {
+      "code": "VN-04",
+      "name": "Cao Bằng",
+      "type": "Province"
+    },
+    {
+      "code": "VN-05",
+      "name": "Sơn La",
+      "type": "Province"
+    },
+    {
+      "code": "VN-06",
+      "name": "Yên Bái",
+      "type": "Province"
+    },
+    {
+      "code": "VN-07",
+      "name": "Tuyên Quang",
+      "type": "Province"
+    },
+    {
+      "code": "VN-09",
+      "name": "Lạng Sơn",
+      "type": "Province"
+    },
+    {
+      "code": "VN-13",
+      "name": "Quảng Ninh",
+      "type": "Province"
+    },
+    {
+      "code": "VN-14",
+      "name": "Hòa Bình",
+      "type": "Province"
+    },
+    {
+      "code": "VN-18",
+      "name": "Ninh Bình",
+      "type": "Province"
+    },
+    {
+      "code": "VN-20",
+      "name": "Thái Bình",
+      "type": "Province"
+    },
+    {
+      "code": "VN-21",
+      "name": "Thanh Hóa",
+      "type": "Province"
+    },
+    {
+      "code": "VN-22",
+      "name": "Nghệ An",
+      "type": "Province"
+    },
+    {
+      "code": "VN-23",
+      "name": "Hà Tĩnh",
+      "type": "Province"
+    },
+    {
+      "code": "VN-24",
+      "name": "Quảng Bình",
+      "type": "Province"
+    },
+    {
+      "code": "VN-25",
+      "name": "Quảng Trị",
+      "type": "Province"
+    },
+    {
+      "code": "VN-26",
+      "name": "Thừa Thiên-Huế",
+      "type": "Province"
+    },
+    {
+      "code": "VN-27",
+      "name": "Quảng Nam",
+      "type": "Province"
+    },
+    {
+      "code": "VN-28",
+      "name": "Kon Tum",
+      "type": "Province"
+    },
+    {
+      "code": "VN-29",
+      "name": "Quảng Ngãi",
+      "type": "Province"
+    },
+    {
+      "code": "VN-30",
+      "name": "Gia Lai",
+      "type": "Province"
+    },
+    {
+      "code": "VN-31",
+      "name": "Bình Định",
+      "type": "Province"
+    },
+    {
+      "code": "VN-32",
+      "name": "Phú Yên",
+      "type": "Province"
+    },
+    {
+      "code": "VN-33",
+      "name": "Đắk Lắk",
+      "type": "Province"
+    },
+    {
+      "code": "VN-34",
+      "name": "Khánh Hòa",
+      "type": "Province"
+    },
+    {
+      "code": "VN-35",
+      "name": "Lâm Đồng",
+      "type": "Province"
+    },
+    {
+      "code": "VN-36",
+      "name": "Ninh Thuận",
+      "type": "Province"
+    },
+    {
+      "code": "VN-37",
+      "name": "Tây Ninh",
+      "type": "Province"
+    },
+    {
+      "code": "VN-39",
+      "name": "Đồng Nai",
+      "type": "Province"
+    },
+    {
+      "code": "VN-40",
+      "name": "Bình Thuận",
+      "type": "Province"
+    },
+    {
+      "code": "VN-41",
+      "name": "Long An",
+      "type": "Province"
+    },
+    {
+      "code": "VN-43",
+      "name": "Bà Rịa - Vũng Tàu",
+      "type": "Province"
+    },
+    {
+      "code": "VN-44",
+      "name": "An Giang",
+      "type": "Province"
+    },
+    {
+      "code": "VN-45",
+      "name": "Đồng Tháp",
+      "type": "Province"
+    },
+    {
+      "code": "VN-46",
+      "name": "Tiền Giang",
+      "type": "Province"
+    },
+    {
+      "code": "VN-47",
+      "name": "Kiến Giang",
+      "type": "Province"
+    },
+    {
+      "code": "VN-49",
+      "name": "Vĩnh Long",
+      "type": "Province"
+    },
+    {
+      "code": "VN-50",
+      "name": "Bến Tre",
+      "type": "Province"
+    },
+    {
+      "code": "VN-51",
+      "name": "Trà Vinh",
+      "type": "Province"
+    },
+    {
+      "code": "VN-52",
+      "name": "Sóc Trăng",
+      "type": "Province"
+    },
+    {
+      "code": "VN-53",
+      "name": "Bắc Kạn",
+      "type": "Province"
+    },
+    {
+      "code": "VN-54",
+      "name": "Bắc Giang",
+      "type": "Province"
+    },
+    {
+      "code": "VN-55",
+      "name": "Bạc Liêu",
+      "type": "Province"
+    },
+    {
+      "code": "VN-56",
+      "name": "Bắc Ninh",
+      "type": "Province"
+    },
+    {
+      "code": "VN-57",
+      "name": "Bình Dương",
+      "type": "Province"
+    },
+    {
+      "code": "VN-58",
+      "name": "Bình Phước",
+      "type": "Province"
+    },
+    {
+      "code": "VN-59",
+      "name": "Cà Mau",
+      "type": "Province"
+    },
+    {
+      "code": "VN-61",
+      "name": "Hải Dương",
+      "type": "Province"
+    },
+    {
+      "code": "VN-63",
+      "name": "Hà Nam",
+      "type": "Province"
+    },
+    {
+      "code": "VN-66",
+      "name": "Hưng Yên",
+      "type": "Province"
+    },
+    {
+      "code": "VN-67",
+      "name": "Nam Định",
+      "type": "Province"
+    },
+    {
+      "code": "VN-68",
+      "name": "Phú Thọ",
+      "type": "Province"
+    },
+    {
+      "code": "VN-69",
+      "name": "Thái Nguyên",
+      "type": "Province"
+    },
+    {
+      "code": "VN-70",
+      "name": "Vĩnh Phúc",
+      "type": "Province"
+    },
+    {
+      "code": "VN-71",
+      "name": "Điện Biên",
+      "type": "Province"
+    },
+    {
+      "code": "VN-72",
+      "name": "Đắk Nông",
+      "type": "Province"
+    },
+    {
+      "code": "VN-73",
+      "name": "Hậu Giang",
+      "type": "Province"
+    },
+    {
+      "code": "VN-CT",
+      "name": "Cần Thơ",
+      "type": "Municipality"
+    },
+    {
+      "code": "VN-DN",
+      "name": "Đà Nẵng",
+      "type": "Municipality"
+    },
+    {
+      "code": "VN-HN",
+      "name": "Hà Nội",
+      "type": "Municipality"
+    },
+    {
+      "code": "VN-HP",
+      "name": "Hải Phòng",
+      "type": "Municipality"
+    },
+    {
+      "code": "VN-SG",
+      "name": "Hồ Chí Minh",
+      "type": "Municipality"
+    },
+    {
+      "code": "VU-MAP",
+      "name": "Malampa",
+      "type": "Province"
+    },
+    {
+      "code": "VU-PAM",
+      "name": "Pénama",
+      "type": "Province"
+    },
+    {
+      "code": "VU-SAM",
+      "name": "Sanma",
+      "type": "Province"
+    },
+    {
+      "code": "VU-SEE",
+      "name": "Shéfa",
+      "type": "Province"
+    },
+    {
+      "code": "VU-TAE",
+      "name": "Taféa",
+      "type": "Province"
+    },
+    {
+      "code": "VU-TOB",
+      "name": "Torba",
+      "type": "Province"
+    },
+    {
+      "code": "WF-AL",
+      "name": "Alo",
+      "type": "Administrative precinct"
+    },
+    {
+      "code": "WF-SG",
+      "name": "Sigave",
+      "type": "Administrative precinct"
+    },
+    {
+      "code": "WF-UV",
+      "name": "Uvea",
+      "type": "Administrative precinct"
+    },
+    {
+      "code": "WS-AA",
+      "name": "A'ana",
+      "type": "District"
+    },
+    {
+      "code": "WS-AL",
+      "name": "Aiga-i-le-Tai",
+      "type": "District"
+    },
+    {
+      "code": "WS-AT",
+      "name": "Atua",
+      "type": "District"
+    },
+    {
+      "code": "WS-FA",
+      "name": "Fa'asaleleaga",
+      "type": "District"
+    },
+    {
+      "code": "WS-GE",
+      "name": "Gaga'emauga",
+      "type": "District"
+    },
+    {
+      "code": "WS-GI",
+      "name": "Gagaifomauga",
+      "type": "District"
+    },
+    {
+      "code": "WS-PA",
+      "name": "Palauli",
+      "type": "District"
+    },
+    {
+      "code": "WS-SA",
+      "name": "Satupa'itea",
+      "type": "District"
+    },
+    {
+      "code": "WS-TU",
+      "name": "Tuamasaga",
+      "type": "District"
+    },
+    {
+      "code": "WS-VF",
+      "name": "Va'a-o-Fonoti",
+      "type": "District"
+    },
+    {
+      "code": "WS-VS",
+      "name": "Vaisigano",
+      "type": "District"
+    },
+    {
+      "code": "YE-AB",
+      "name": "Abyan",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-AD",
+      "name": "‘Adan",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-AM",
+      "name": "‘Amrān",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-BA",
+      "name": "Al Bayḑā’",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-DA",
+      "name": "Aḑ Ḑāli‘",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-DH",
+      "name": "Dhamār",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-HD",
+      "name": "Ḩaḑramawt",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-HJ",
+      "name": "Ḩajjah",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-HU",
+      "name": "Al Ḩudaydah",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-IB",
+      "name": "Ibb",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-JA",
+      "name": "Al Jawf",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-LA",
+      "name": "Laḩij",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-MA",
+      "name": "Ma’rib",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-MR",
+      "name": "Al Mahrah",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-MW",
+      "name": "Al Maḩwīt",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-RA",
+      "name": "Raymah",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-SA",
+      "name": "Amānat al ‘Āşimah [city]",
+      "type": "Municipality"
+    },
+    {
+      "code": "YE-SD",
+      "name": "Şāʻdah",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-SH",
+      "name": "Shabwah",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-SN",
+      "name": "Şanʻā’",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-SU",
+      "name": "Arkhabīl Suquţrá",
+      "type": "Governorate"
+    },
+    {
+      "code": "YE-TA",
+      "name": "Tāʻizz",
+      "type": "Governorate"
+    },
+    {
+      "code": "ZA-EC",
+      "name": "Eastern Cape",
+      "type": "Province"
+    },
+    {
+      "code": "ZA-FS",
+      "name": "Free State",
+      "type": "Province"
+    },
+    {
+      "code": "ZA-GP",
+      "name": "Gauteng",
+      "type": "Province"
+    },
+    {
+      "code": "ZA-KZN",
+      "name": "Kwazulu-Natal",
+      "type": "Province"
+    },
+    {
+      "code": "ZA-LP",
+      "name": "Limpopo",
+      "type": "Province"
+    },
+    {
+      "code": "ZA-MP",
+      "name": "Mpumalanga",
+      "type": "Province"
+    },
+    {
+      "code": "ZA-NC",
+      "name": "Northern Cape",
+      "type": "Province"
+    },
+    {
+      "code": "ZA-NW",
+      "name": "North-West",
+      "type": "Province"
+    },
+    {
+      "code": "ZA-WC",
+      "name": "Western Cape",
+      "type": "Province"
+    },
+    {
+      "code": "ZM-01",
+      "name": "Western",
+      "type": "Province"
+    },
+    {
+      "code": "ZM-02",
+      "name": "Central",
+      "type": "Province"
+    },
+    {
+      "code": "ZM-03",
+      "name": "Eastern",
+      "type": "Province"
+    },
+    {
+      "code": "ZM-04",
+      "name": "Luapula",
+      "type": "Province"
+    },
+    {
+      "code": "ZM-05",
+      "name": "Northern",
+      "type": "Province"
+    },
+    {
+      "code": "ZM-06",
+      "name": "North-Western",
+      "type": "Province"
+    },
+    {
+      "code": "ZM-07",
+      "name": "Southern",
+      "type": "Province"
+    },
+    {
+      "code": "ZM-08",
+      "name": "Copperbelt",
+      "type": "Province"
+    },
+    {
+      "code": "ZM-09",
+      "name": "Lusaka",
+      "type": "Province"
+    },
+    {
+      "code": "ZM-10",
+      "name": "Muchinga",
+      "type": "Province"
+    },
+    {
+      "code": "ZW-BU",
+      "name": "Bulawayo",
+      "type": "Province"
+    },
+    {
+      "code": "ZW-HA",
+      "name": "Harare",
+      "type": "Province"
+    },
+    {
+      "code": "ZW-MA",
+      "name": "Manicaland",
+      "type": "Province"
+    },
+    {
+      "code": "ZW-MC",
+      "name": "Mashonaland Central",
+      "type": "Province"
+    },
+    {
+      "code": "ZW-ME",
+      "name": "Mashonaland East",
+      "type": "Province"
+    },
+    {
+      "code": "ZW-MI",
+      "name": "Midlands",
+      "type": "Province"
+    },
+    {
+      "code": "ZW-MN",
+      "name": "Matabeleland North",
+      "type": "Province"
+    },
+    {
+      "code": "ZW-MS",
+      "name": "Matabeleland South",
+      "type": "Province"
+    },
+    {
+      "code": "ZW-MV",
+      "name": "Masvingo",
+      "type": "Province"
+    },
+    {
+      "code": "ZW-MW",
+      "name": "Mashonaland West",
+      "type": "Province"
+    }
+  ]
+}
diff --git a/lib/python3.1/site-packages/pycountry/databases/iso3166-3.json b/lib/python3.1/site-packages/pycountry/databases/iso3166-3.json
new file mode 100644
index 0000000000000000000000000000000000000000..88b462881e617b7a3206a6d8a6b3e910682bbc16
--- /dev/null
+++ b/lib/python3.1/site-packages/pycountry/databases/iso3166-3.json
@@ -0,0 +1,254 @@
+{
+  "3166-3": [
+    {
+      "alpha_2": "AI",
+      "alpha_3": "AFI",
+      "alpha_4": "AIDJ",
+      "name": "French Afars and Issas",
+      "numeric": "262",
+      "withdrawal_date": "1977"
+    },
+    {
+      "alpha_2": "AN",
+      "alpha_3": "ANT",
+      "alpha_4": "ANHH",
+      "comment": "had numeric code 532 until Aruba split away in 1986",
+      "name": "Netherlands Antilles",
+      "numeric": "530",
+      "withdrawal_date": "2010-12-15"
+    },
+    {
+      "alpha_2": "BQ",
+      "alpha_3": "ATB",
+      "alpha_4": "BQAQ",
+      "name": "British Antarctic Territory",
+      "withdrawal_date": "1979"
+    },
+    {
+      "alpha_2": "BU",
+      "alpha_3": "BUR",
+      "alpha_4": "BUMM",
+      "name": "Burma, Socialist Republic of the Union of",
+      "numeric": "104",
+      "withdrawal_date": "1989-12-05"
+    },
+    {
+      "alpha_2": "BY",
+      "alpha_3": "BYS",
+      "alpha_4": "BYAA",
+      "name": "Byelorussian SSR Soviet Socialist Republic",
+      "numeric": "112",
+      "withdrawal_date": "1992-06-15"
+    },
+    {
+      "alpha_2": "CS",
+      "alpha_3": "CSK",
+      "alpha_4": "CSHH",
+      "name": "Czechoslovakia, Czechoslovak Socialist Republic",
+      "numeric": "200",
+      "withdrawal_date": "1993-06-15"
+    },
+    {
+      "alpha_2": "CS",
+      "alpha_3": "SCG",
+      "alpha_4": "CSXX",
+      "name": "Serbia and Montenegro",
+      "numeric": "891",
+      "withdrawal_date": "2006-09-26"
+    },
+    {
+      "alpha_2": "CT",
+      "alpha_3": "CTE",
+      "alpha_4": "CTKI",
+      "name": "Canton and Enderbury Islands",
+      "numeric": "128",
+      "withdrawal_date": "1984"
+    },
+    {
+      "alpha_2": "DD",
+      "alpha_3": "DDR",
+      "alpha_4": "DDDE",
+      "name": "German Democratic Republic",
+      "numeric": "278",
+      "withdrawal_date": "1990-10-30"
+    },
+    {
+      "alpha_2": "DY",
+      "alpha_3": "DHY",
+      "alpha_4": "DYBJ",
+      "name": "Dahomey",
+      "numeric": "204",
+      "withdrawal_date": "1977"
+    },
+    {
+      "alpha_2": "FQ",
+      "alpha_3": "ATF",
+      "alpha_4": "FQHH",
+      "comment": "now split between AQ and TF",
+      "name": "French Southern and Antarctic Territories",
+      "withdrawal_date": "1979"
+    },
+    {
+      "alpha_2": "FX",
+      "alpha_3": "FXX",
+      "alpha_4": "FXFR",
+      "name": "France, Metropolitan",
+      "numeric": "249",
+      "withdrawal_date": "1997-07-14"
+    },
+    {
+      "alpha_2": "GE",
+      "alpha_3": "GEL",
+      "alpha_4": "GEHH",
+      "comment": "now split into Kiribati and Tuvalu",
+      "name": "Gilbert and Ellice Islands",
+      "numeric": "296",
+      "withdrawal_date": "1979"
+    },
+    {
+      "alpha_2": "HV",
+      "alpha_3": "HVO",
+      "alpha_4": "HVBF",
+      "name": "Upper Volta, Republic of",
+      "numeric": "854",
+      "withdrawal_date": "1984"
+    },
+    {
+      "alpha_2": "JT",
+      "alpha_3": "JTN",
+      "alpha_4": "JTUM",
+      "name": "Johnston Island",
+      "numeric": "396",
+      "withdrawal_date": "1986"
+    },
+    {
+      "alpha_2": "MI",
+      "alpha_3": "MID",
+      "alpha_4": "MIUM",
+      "name": "Midway Islands",
+      "numeric": "488",
+      "withdrawal_date": "1986"
+    },
+    {
+      "alpha_2": "NH",
+      "alpha_3": "NHB",
+      "alpha_4": "NHVU",
+      "name": "New Hebrides",
+      "numeric": "548",
+      "withdrawal_date": "1980"
+    },
+    {
+      "alpha_2": "NQ",
+      "alpha_3": "ATN",
+      "alpha_4": "NQAQ",
+      "name": "Dronning Maud Land",
+      "numeric": "216",
+      "withdrawal_date": "1983"
+    },
+    {
+      "alpha_2": "NT",
+      "alpha_3": "NTZ",
+      "alpha_4": "NTHH",
+      "comment": "formerly between Saudi Arabia and Iraq",
+      "name": "Neutral Zone",
+      "numeric": "536",
+      "withdrawal_date": "1993-07-12"
+    },
+    {
+      "alpha_2": "PC",
+      "alpha_3": "PCI",
+      "alpha_4": "PCHH",
+      "comment": "divided into FM, MH, MP, and PW",
+      "name": "Pacific Islands (trust territory)",
+      "numeric": "582",
+      "withdrawal_date": "1986"
+    },
+    {
+      "alpha_2": "PU",
+      "alpha_3": "PUS",
+      "alpha_4": "PUUM",
+      "name": "US Miscellaneous Pacific Islands",
+      "numeric": "849",
+      "withdrawal_date": "1986"
+    },
+    {
+      "alpha_2": "PZ",
+      "alpha_3": "PCZ",
+      "alpha_4": "PZPA",
+      "name": "Panama Canal Zone",
+      "withdrawal_date": "1980"
+    },
+    {
+      "alpha_2": "RH",
+      "alpha_3": "RHO",
+      "alpha_4": "RHZW",
+      "name": "Southern Rhodesia",
+      "numeric": "716",
+      "withdrawal_date": "1980"
+    },
+    {
+      "alpha_2": "SK",
+      "alpha_3": "SKM",
+      "alpha_4": "SKIN",
+      "name": "Sikkim",
+      "withdrawal_date": "1975"
+    },
+    {
+      "alpha_2": "SU",
+      "alpha_3": "SUN",
+      "alpha_4": "SUHH",
+      "name": "USSR, Union of Soviet Socialist Republics",
+      "numeric": "810",
+      "withdrawal_date": "1992-08-30"
+    },
+    {
+      "alpha_2": "TP",
+      "alpha_3": "TMP",
+      "alpha_4": "TPTL",
+      "comment": "was Portuguese Timor",
+      "name": "East Timor",
+      "numeric": "626",
+      "withdrawal_date": "2002-05-20"
+    },
+    {
+      "alpha_2": "VD",
+      "alpha_3": "VDR",
+      "alpha_4": "VDVN",
+      "name": "Viet-Nam, Democratic Republic of",
+      "withdrawal_date": "1977"
+    },
+    {
+      "alpha_2": "WK",
+      "alpha_3": "WAK",
+      "alpha_4": "WKUM",
+      "name": "Wake Island",
+      "numeric": "872",
+      "withdrawal_date": "1986"
+    },
+    {
+      "alpha_2": "YD",
+      "alpha_3": "YMD",
+      "alpha_4": "YDYE",
+      "name": "Yemen, Democratic, People's Democratic Republic of",
+      "numeric": "720",
+      "withdrawal_date": "1990-08-14"
+    },
+    {
+      "alpha_2": "YU",
+      "alpha_3": "YUG",
+      "alpha_4": "YUCS",
+      "comment": "had numeric code 890 until the 'Socialist Federal Republic of Yugoslavia' formerly broke apart on 27 April 1992 and the 'Federal Republic of Yugoslavia' was founded",
+      "name": "Yugoslavia, (Socialist) Federal Republic of",
+      "numeric": "891",
+      "withdrawal_date": "2003-07-23"
+    },
+    {
+      "alpha_2": "ZR",
+      "alpha_3": "ZAR",
+      "alpha_4": "ZRCD",
+      "name": "Zaire, Republic of",
+      "numeric": "180",
+      "withdrawal_date": "1997-07-14"
+    }
+  ]
+}
diff --git a/lib/python3.1/site-packages/pycountry/databases/iso4217.json b/lib/python3.1/site-packages/pycountry/databases/iso4217.json
new file mode 100644
index 0000000000000000000000000000000000000000..4bd42d3bc7cce146d422098eeb88583a59681448
--- /dev/null
+++ b/lib/python3.1/site-packages/pycountry/databases/iso4217.json
@@ -0,0 +1,909 @@
+{
+  "4217": [
+    {
+      "alpha_3": "AED",
+      "name": "UAE Dirham",
+      "numeric": "784"
+    },
+    {
+      "alpha_3": "AFN",
+      "name": "Afghani",
+      "numeric": "971"
+    },
+    {
+      "alpha_3": "ALL",
+      "name": "Lek",
+      "numeric": "008"
+    },
+    {
+      "alpha_3": "AMD",
+      "name": "Armenian Dram",
+      "numeric": "051"
+    },
+    {
+      "alpha_3": "ANG",
+      "name": "Netherlands Antillean Guilder",
+      "numeric": "532"
+    },
+    {
+      "alpha_3": "AOA",
+      "name": "Kwanza",
+      "numeric": "973"
+    },
+    {
+      "alpha_3": "ARS",
+      "name": "Argentine Peso",
+      "numeric": "032"
+    },
+    {
+      "alpha_3": "AUD",
+      "name": "Australian Dollar",
+      "numeric": "036"
+    },
+    {
+      "alpha_3": "AWG",
+      "name": "Aruban Florin",
+      "numeric": "533"
+    },
+    {
+      "alpha_3": "AZN",
+      "name": "Azerbaijan Manat",
+      "numeric": "944"
+    },
+    {
+      "alpha_3": "BAM",
+      "name": "Convertible Mark",
+      "numeric": "977"
+    },
+    {
+      "alpha_3": "BBD",
+      "name": "Barbados Dollar",
+      "numeric": "052"
+    },
+    {
+      "alpha_3": "BDT",
+      "name": "Taka",
+      "numeric": "050"
+    },
+    {
+      "alpha_3": "BGN",
+      "name": "Bulgarian Lev",
+      "numeric": "975"
+    },
+    {
+      "alpha_3": "BHD",
+      "name": "Bahraini Dinar",
+      "numeric": "048"
+    },
+    {
+      "alpha_3": "BIF",
+      "name": "Burundi Franc",
+      "numeric": "108"
+    },
+    {
+      "alpha_3": "BMD",
+      "name": "Bermudian Dollar",
+      "numeric": "060"
+    },
+    {
+      "alpha_3": "BND",
+      "name": "Brunei Dollar",
+      "numeric": "096"
+    },
+    {
+      "alpha_3": "BOB",
+      "name": "Boliviano",
+      "numeric": "068"
+    },
+    {
+      "alpha_3": "BOV",
+      "name": "Mvdol",
+      "numeric": "984"
+    },
+    {
+      "alpha_3": "BRL",
+      "name": "Brazilian Real",
+      "numeric": "986"
+    },
+    {
+      "alpha_3": "BSD",
+      "name": "Bahamian Dollar",
+      "numeric": "044"
+    },
+    {
+      "alpha_3": "BTN",
+      "name": "Ngultrum",
+      "numeric": "064"
+    },
+    {
+      "alpha_3": "BWP",
+      "name": "Pula",
+      "numeric": "072"
+    },
+    {
+      "alpha_3": "BYN",
+      "name": "Belarusian Ruble",
+      "numeric": "933"
+    },
+    {
+      "alpha_3": "BZD",
+      "name": "Belize Dollar",
+      "numeric": "084"
+    },
+    {
+      "alpha_3": "CAD",
+      "name": "Canadian Dollar",
+      "numeric": "124"
+    },
+    {
+      "alpha_3": "CDF",
+      "name": "Congolese Franc",
+      "numeric": "976"
+    },
+    {
+      "alpha_3": "CHE",
+      "name": "WIR Euro",
+      "numeric": "947"
+    },
+    {
+      "alpha_3": "CHF",
+      "name": "Swiss Franc",
+      "numeric": "756"
+    },
+    {
+      "alpha_3": "CHW",
+      "name": "WIR Franc",
+      "numeric": "948"
+    },
+    {
+      "alpha_3": "CLF",
+      "name": "Unidad de Fomento",
+      "numeric": "990"
+    },
+    {
+      "alpha_3": "CLP",
+      "name": "Chilean Peso",
+      "numeric": "152"
+    },
+    {
+      "alpha_3": "CNY",
+      "name": "Yuan Renminbi",
+      "numeric": "156"
+    },
+    {
+      "alpha_3": "COP",
+      "name": "Colombian Peso",
+      "numeric": "170"
+    },
+    {
+      "alpha_3": "COU",
+      "name": "Unidad de Valor Real",
+      "numeric": "970"
+    },
+    {
+      "alpha_3": "CRC",
+      "name": "Costa Rican Colon",
+      "numeric": "188"
+    },
+    {
+      "alpha_3": "CUC",
+      "name": "Peso Convertible",
+      "numeric": "931"
+    },
+    {
+      "alpha_3": "CUP",
+      "name": "Cuban Peso",
+      "numeric": "192"
+    },
+    {
+      "alpha_3": "CVE",
+      "name": "Cabo Verde Escudo",
+      "numeric": "132"
+    },
+    {
+      "alpha_3": "CZK",
+      "name": "Czech Koruna",
+      "numeric": "203"
+    },
+    {
+      "alpha_3": "DJF",
+      "name": "Djibouti Franc",
+      "numeric": "262"
+    },
+    {
+      "alpha_3": "DKK",
+      "name": "Danish Krone",
+      "numeric": "208"
+    },
+    {
+      "alpha_3": "DOP",
+      "name": "Dominican Peso",
+      "numeric": "214"
+    },
+    {
+      "alpha_3": "DZD",
+      "name": "Algerian Dinar",
+      "numeric": "012"
+    },
+    {
+      "alpha_3": "EGP",
+      "name": "Egyptian Pound",
+      "numeric": "818"
+    },
+    {
+      "alpha_3": "ERN",
+      "name": "Nakfa",
+      "numeric": "232"
+    },
+    {
+      "alpha_3": "ETB",
+      "name": "Ethiopian Birr",
+      "numeric": "230"
+    },
+    {
+      "alpha_3": "EUR",
+      "name": "Euro",
+      "numeric": "978"
+    },
+    {
+      "alpha_3": "FJD",
+      "name": "Fiji Dollar",
+      "numeric": "242"
+    },
+    {
+      "alpha_3": "FKP",
+      "name": "Falkland Islands Pound",
+      "numeric": "238"
+    },
+    {
+      "alpha_3": "GBP",
+      "name": "Pound Sterling",
+      "numeric": "826"
+    },
+    {
+      "alpha_3": "GEL",
+      "name": "Lari",
+      "numeric": "981"
+    },
+    {
+      "alpha_3": "GHS",
+      "name": "Ghana Cedi",
+      "numeric": "936"
+    },
+    {
+      "alpha_3": "GIP",
+      "name": "Gibraltar Pound",
+      "numeric": "292"
+    },
+    {
+      "alpha_3": "GMD",
+      "name": "Dalasi",
+      "numeric": "270"
+    },
+    {
+      "alpha_3": "GNF",
+      "name": "Guinean Franc",
+      "numeric": "324"
+    },
+    {
+      "alpha_3": "GTQ",
+      "name": "Quetzal",
+      "numeric": "320"
+    },
+    {
+      "alpha_3": "GYD",
+      "name": "Guyana Dollar",
+      "numeric": "328"
+    },
+    {
+      "alpha_3": "HKD",
+      "name": "Hong Kong Dollar",
+      "numeric": "344"
+    },
+    {
+      "alpha_3": "HNL",
+      "name": "Lempira",
+      "numeric": "340"
+    },
+    {
+      "alpha_3": "HRK",
+      "name": "Kuna",
+      "numeric": "191"
+    },
+    {
+      "alpha_3": "HTG",
+      "name": "Gourde",
+      "numeric": "332"
+    },
+    {
+      "alpha_3": "HUF",
+      "name": "Forint",
+      "numeric": "348"
+    },
+    {
+      "alpha_3": "IDR",
+      "name": "Rupiah",
+      "numeric": "360"
+    },
+    {
+      "alpha_3": "ILS",
+      "name": "New Israeli Sheqel",
+      "numeric": "376"
+    },
+    {
+      "alpha_3": "INR",
+      "name": "Indian Rupee",
+      "numeric": "356"
+    },
+    {
+      "alpha_3": "IQD",
+      "name": "Iraqi Dinar",
+      "numeric": "368"
+    },
+    {
+      "alpha_3": "IRR",
+      "name": "Iranian Rial",
+      "numeric": "364"
+    },
+    {
+      "alpha_3": "ISK",
+      "name": "Iceland Krona",
+      "numeric": "352"
+    },
+    {
+      "alpha_3": "JMD",
+      "name": "Jamaican Dollar",
+      "numeric": "388"
+    },
+    {
+      "alpha_3": "JOD",
+      "name": "Jordanian Dinar",
+      "numeric": "400"
+    },
+    {
+      "alpha_3": "JPY",
+      "name": "Yen",
+      "numeric": "392"
+    },
+    {
+      "alpha_3": "KES",
+      "name": "Kenyan Shilling",
+      "numeric": "404"
+    },
+    {
+      "alpha_3": "KGS",
+      "name": "Som",
+      "numeric": "417"
+    },
+    {
+      "alpha_3": "KHR",
+      "name": "Riel",
+      "numeric": "116"
+    },
+    {
+      "alpha_3": "KMF",
+      "name": "Comorian Franc",
+      "numeric": "174"
+    },
+    {
+      "alpha_3": "KPW",
+      "name": "North Korean Won",
+      "numeric": "408"
+    },
+    {
+      "alpha_3": "KRW",
+      "name": "Won",
+      "numeric": "410"
+    },
+    {
+      "alpha_3": "KWD",
+      "name": "Kuwaiti Dinar",
+      "numeric": "414"
+    },
+    {
+      "alpha_3": "KYD",
+      "name": "Cayman Islands Dollar",
+      "numeric": "136"
+    },
+    {
+      "alpha_3": "KZT",
+      "name": "Tenge",
+      "numeric": "398"
+    },
+    {
+      "alpha_3": "LAK",
+      "name": "Lao Kip",
+      "numeric": "418"
+    },
+    {
+      "alpha_3": "LBP",
+      "name": "Lebanese Pound",
+      "numeric": "422"
+    },
+    {
+      "alpha_3": "LKR",
+      "name": "Sri Lanka Rupee",
+      "numeric": "144"
+    },
+    {
+      "alpha_3": "LRD",
+      "name": "Liberian Dollar",
+      "numeric": "430"
+    },
+    {
+      "alpha_3": "LSL",
+      "name": "Loti",
+      "numeric": "426"
+    },
+    {
+      "alpha_3": "LYD",
+      "name": "Libyan Dinar",
+      "numeric": "434"
+    },
+    {
+      "alpha_3": "MAD",
+      "name": "Moroccan Dirham",
+      "numeric": "504"
+    },
+    {
+      "alpha_3": "MDL",
+      "name": "Moldovan Leu",
+      "numeric": "498"
+    },
+    {
+      "alpha_3": "MGA",
+      "name": "Malagasy Ariary",
+      "numeric": "969"
+    },
+    {
+      "alpha_3": "MKD",
+      "name": "Denar",
+      "numeric": "807"
+    },
+    {
+      "alpha_3": "MMK",
+      "name": "Kyat",
+      "numeric": "104"
+    },
+    {
+      "alpha_3": "MNT",
+      "name": "Tugrik",
+      "numeric": "496"
+    },
+    {
+      "alpha_3": "MOP",
+      "name": "Pataca",
+      "numeric": "446"
+    },
+    {
+      "alpha_3": "MRU",
+      "name": "Ouguiya",
+      "numeric": "929"
+    },
+    {
+      "alpha_3": "MUR",
+      "name": "Mauritius Rupee",
+      "numeric": "480"
+    },
+    {
+      "alpha_3": "MVR",
+      "name": "Rufiyaa",
+      "numeric": "462"
+    },
+    {
+      "alpha_3": "MWK",
+      "name": "Malawi Kwacha",
+      "numeric": "454"
+    },
+    {
+      "alpha_3": "MXN",
+      "name": "Mexican Peso",
+      "numeric": "484"
+    },
+    {
+      "alpha_3": "MXV",
+      "name": "Mexican Unidad de Inversion (UDI)",
+      "numeric": "979"
+    },
+    {
+      "alpha_3": "MYR",
+      "name": "Malaysian Ringgit",
+      "numeric": "458"
+    },
+    {
+      "alpha_3": "MZN",
+      "name": "Mozambique Metical",
+      "numeric": "943"
+    },
+    {
+      "alpha_3": "NAD",
+      "name": "Namibia Dollar",
+      "numeric": "516"
+    },
+    {
+      "alpha_3": "NGN",
+      "name": "Naira",
+      "numeric": "566"
+    },
+    {
+      "alpha_3": "NIO",
+      "name": "Cordoba Oro",
+      "numeric": "558"
+    },
+    {
+      "alpha_3": "NOK",
+      "name": "Norwegian Krone",
+      "numeric": "578"
+    },
+    {
+      "alpha_3": "NPR",
+      "name": "Nepalese Rupee",
+      "numeric": "524"
+    },
+    {
+      "alpha_3": "NZD",
+      "name": "New Zealand Dollar",
+      "numeric": "554"
+    },
+    {
+      "alpha_3": "OMR",
+      "name": "Rial Omani",
+      "numeric": "512"
+    },
+    {
+      "alpha_3": "PAB",
+      "name": "Balboa",
+      "numeric": "590"
+    },
+    {
+      "alpha_3": "PEN",
+      "name": "Sol",
+      "numeric": "604"
+    },
+    {
+      "alpha_3": "PGK",
+      "name": "Kina",
+      "numeric": "598"
+    },
+    {
+      "alpha_3": "PHP",
+      "name": "Philippine Peso",
+      "numeric": "608"
+    },
+    {
+      "alpha_3": "PKR",
+      "name": "Pakistan Rupee",
+      "numeric": "586"
+    },
+    {
+      "alpha_3": "PLN",
+      "name": "Zloty",
+      "numeric": "985"
+    },
+    {
+      "alpha_3": "PYG",
+      "name": "Guarani",
+      "numeric": "600"
+    },
+    {
+      "alpha_3": "QAR",
+      "name": "Qatari Rial",
+      "numeric": "634"
+    },
+    {
+      "alpha_3": "RON",
+      "name": "Romanian Leu",
+      "numeric": "946"
+    },
+    {
+      "alpha_3": "RSD",
+      "name": "Serbian Dinar",
+      "numeric": "941"
+    },
+    {
+      "alpha_3": "RUB",
+      "name": "Russian Ruble",
+      "numeric": "643"
+    },
+    {
+      "alpha_3": "RWF",
+      "name": "Rwanda Franc",
+      "numeric": "646"
+    },
+    {
+      "alpha_3": "SAR",
+      "name": "Saudi Riyal",
+      "numeric": "682"
+    },
+    {
+      "alpha_3": "SBD",
+      "name": "Solomon Islands Dollar",
+      "numeric": "090"
+    },
+    {
+      "alpha_3": "SCR",
+      "name": "Seychelles Rupee",
+      "numeric": "690"
+    },
+    {
+      "alpha_3": "SDG",
+      "name": "Sudanese Pound",
+      "numeric": "938"
+    },
+    {
+      "alpha_3": "SEK",
+      "name": "Swedish Krona",
+      "numeric": "752"
+    },
+    {
+      "alpha_3": "SGD",
+      "name": "Singapore Dollar",
+      "numeric": "702"
+    },
+    {
+      "alpha_3": "SHP",
+      "name": "Saint Helena Pound",
+      "numeric": "654"
+    },
+    {
+      "alpha_3": "SLE",
+      "name": "Leone",
+      "numeric": "925"
+    },
+    {
+      "alpha_3": "SLL",
+      "name": "Leone",
+      "numeric": "694"
+    },
+    {
+      "alpha_3": "SOS",
+      "name": "Somali Shilling",
+      "numeric": "706"
+    },
+    {
+      "alpha_3": "SRD",
+      "name": "Surinam Dollar",
+      "numeric": "968"
+    },
+    {
+      "alpha_3": "SSP",
+      "name": "South Sudanese Pound",
+      "numeric": "728"
+    },
+    {
+      "alpha_3": "STN",
+      "name": "Dobra",
+      "numeric": "930"
+    },
+    {
+      "alpha_3": "SVC",
+      "name": "El Salvador Colon",
+      "numeric": "222"
+    },
+    {
+      "alpha_3": "SYP",
+      "name": "Syrian Pound",
+      "numeric": "760"
+    },
+    {
+      "alpha_3": "SZL",
+      "name": "Lilangeni",
+      "numeric": "748"
+    },
+    {
+      "alpha_3": "THB",
+      "name": "Baht",
+      "numeric": "764"
+    },
+    {
+      "alpha_3": "TJS",
+      "name": "Somoni",
+      "numeric": "972"
+    },
+    {
+      "alpha_3": "TMT",
+      "name": "Turkmenistan New Manat",
+      "numeric": "934"
+    },
+    {
+      "alpha_3": "TND",
+      "name": "Tunisian Dinar",
+      "numeric": "788"
+    },
+    {
+      "alpha_3": "TOP",
+      "name": "Pa’anga",
+      "numeric": "776"
+    },
+    {
+      "alpha_3": "TRY",
+      "name": "Turkish Lira",
+      "numeric": "949"
+    },
+    {
+      "alpha_3": "TTD",
+      "name": "Trinidad and Tobago Dollar",
+      "numeric": "780"
+    },
+    {
+      "alpha_3": "TWD",
+      "name": "New Taiwan Dollar",
+      "numeric": "901"
+    },
+    {
+      "alpha_3": "TZS",
+      "name": "Tanzanian Shilling",
+      "numeric": "834"
+    },
+    {
+      "alpha_3": "UAH",
+      "name": "Hryvnia",
+      "numeric": "980"
+    },
+    {
+      "alpha_3": "UGX",
+      "name": "Uganda Shilling",
+      "numeric": "800"
+    },
+    {
+      "alpha_3": "USD",
+      "name": "US Dollar",
+      "numeric": "840"
+    },
+    {
+      "alpha_3": "USN",
+      "name": "US Dollar (Next day)",
+      "numeric": "997"
+    },
+    {
+      "alpha_3": "UYI",
+      "name": "Uruguay Peso en Unidades Indexadas (UI)",
+      "numeric": "940"
+    },
+    {
+      "alpha_3": "UYU",
+      "name": "Peso Uruguayo",
+      "numeric": "858"
+    },
+    {
+      "alpha_3": "UYW",
+      "name": "Unidad Previsional",
+      "numeric": "927"
+    },
+    {
+      "alpha_3": "UZS",
+      "name": "Uzbekistan Sum",
+      "numeric": "860"
+    },
+    {
+      "alpha_3": "VED",
+      "name": "Bolívar Soberano",
+      "numeric": "926"
+    },
+    {
+      "alpha_3": "VES",
+      "name": "Bolívar Soberano",
+      "numeric": "928"
+    },
+    {
+      "alpha_3": "VND",
+      "name": "Dong",
+      "numeric": "704"
+    },
+    {
+      "alpha_3": "VUV",
+      "name": "Vatu",
+      "numeric": "548"
+    },
+    {
+      "alpha_3": "WST",
+      "name": "Tala",
+      "numeric": "882"
+    },
+    {
+      "alpha_3": "XAF",
+      "name": "CFA Franc BEAC",
+      "numeric": "950"
+    },
+    {
+      "alpha_3": "XAG",
+      "name": "Silver",
+      "numeric": "961"
+    },
+    {
+      "alpha_3": "XAU",
+      "name": "Gold",
+      "numeric": "959"
+    },
+    {
+      "alpha_3": "XBA",
+      "name": "Bond Markets Unit European Composite Unit (EURCO)",
+      "numeric": "955"
+    },
+    {
+      "alpha_3": "XBB",
+      "name": "Bond Markets Unit European Monetary Unit (E.M.U.-6)",
+      "numeric": "956"
+    },
+    {
+      "alpha_3": "XBC",
+      "name": "Bond Markets Unit European Unit of Account 9 (E.U.A.-9)",
+      "numeric": "957"
+    },
+    {
+      "alpha_3": "XBD",
+      "name": "Bond Markets Unit European Unit of Account 17 (E.U.A.-17)",
+      "numeric": "958"
+    },
+    {
+      "alpha_3": "XCD",
+      "name": "East Caribbean Dollar",
+      "numeric": "951"
+    },
+    {
+      "alpha_3": "XDR",
+      "name": "SDR (Special Drawing Right)",
+      "numeric": "960"
+    },
+    {
+      "alpha_3": "XOF",
+      "name": "CFA Franc BCEAO",
+      "numeric": "952"
+    },
+    {
+      "alpha_3": "XPD",
+      "name": "Palladium",
+      "numeric": "964"
+    },
+    {
+      "alpha_3": "XPF",
+      "name": "CFP Franc",
+      "numeric": "953"
+    },
+    {
+      "alpha_3": "XPT",
+      "name": "Platinum",
+      "numeric": "962"
+    },
+    {
+      "alpha_3": "XSU",
+      "name": "Sucre",
+      "numeric": "994"
+    },
+    {
+      "alpha_3": "XTS",
+      "name": "Codes specifically reserved for testing purposes",
+      "numeric": "963"
+    },
+    {
+      "alpha_3": "XUA",
+      "name": "ADB Unit of Account",
+      "numeric": "965"
+    },
+    {
+      "alpha_3": "XXX",
+      "name": "The codes assigned for transactions where no currency is involved",
+      "numeric": "999"
+    },
+    {
+      "alpha_3": "YER",
+      "name": "Yemeni Rial",
+      "numeric": "886"
+    },
+    {
+      "alpha_3": "ZAR",
+      "name": "Rand",
+      "numeric": "710"
+    },
+    {
+      "alpha_3": "ZMW",
+      "name": "Zambian Kwacha",
+      "numeric": "967"
+    },
+    {
+      "alpha_3": "ZWL",
+      "name": "Zimbabwe Dollar",
+      "numeric": "932"
+    }
+  ]
+}
diff --git a/lib/python3.1/site-packages/pycountry/databases/iso639-3.json b/lib/python3.1/site-packages/pycountry/databases/iso639-3.json
new file mode 100644
index 0000000000000000000000000000000000000000..7a09889c1910fa6ee70c9bb12bc35c7ff2561019
--- /dev/null
+++ b/lib/python3.1/site-packages/pycountry/databases/iso639-3.json
@@ -0,0 +1,49084 @@
+{
+  "639-3": [
+    {
+      "alpha_3": "aaa",
+      "name": "Ghotuo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aab",
+      "name": "Alumu-Tesu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aac",
+      "name": "Ari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aad",
+      "name": "Amal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aae",
+      "inverted_name": "Albanian, Arbëreshë",
+      "name": "Arbëreshë Albanian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aaf",
+      "name": "Aranadan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aag",
+      "name": "Ambrak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aah",
+      "inverted_name": "Arapesh, Abu'",
+      "name": "Abu' Arapesh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aai",
+      "name": "Arifama-Miniafia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aak",
+      "name": "Ankave",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aal",
+      "name": "Afade",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aan",
+      "name": "Anambé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aao",
+      "inverted_name": "Arabic, Algerian Saharan",
+      "name": "Algerian Saharan Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aap",
+      "inverted_name": "Arára, Pará",
+      "name": "Pará Arára",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aaq",
+      "inverted_name": "Abnaki, Eastern",
+      "name": "Eastern Abnaki",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_2": "aa",
+      "alpha_3": "aar",
+      "name": "Afar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aas",
+      "name": "Aasáx",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aat",
+      "inverted_name": "Albanian, Arvanitika",
+      "name": "Arvanitika Albanian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aau",
+      "name": "Abau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aaw",
+      "name": "Solong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aax",
+      "name": "Mandobo Atas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aaz",
+      "name": "Amarasi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aba",
+      "name": "Abé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abb",
+      "name": "Bankon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abc",
+      "inverted_name": "Ayta, Ambala",
+      "name": "Ambala Ayta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abd",
+      "name": "Manide",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abe",
+      "inverted_name": "Abnaki, Western",
+      "name": "Western Abnaki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abf",
+      "name": "Abai Sungai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abg",
+      "name": "Abaga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abh",
+      "inverted_name": "Arabic, Tajiki",
+      "name": "Tajiki Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abi",
+      "name": "Abidji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abj",
+      "name": "Aka-Bea",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_2": "ab",
+      "alpha_3": "abk",
+      "name": "Abkhazian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abl",
+      "name": "Lampung Nyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abm",
+      "name": "Abanyom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abn",
+      "name": "Abua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abo",
+      "name": "Abon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abp",
+      "inverted_name": "Ayta, Abellen",
+      "name": "Abellen Ayta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abq",
+      "name": "Abaza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abr",
+      "name": "Abron",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abs",
+      "inverted_name": "Malay, Ambonese",
+      "name": "Ambonese Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abt",
+      "name": "Ambulas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abu",
+      "name": "Abure",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abv",
+      "inverted_name": "Arabic, Baharna",
+      "name": "Baharna Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abw",
+      "name": "Pal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abx",
+      "name": "Inabaknon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aby",
+      "name": "Aneme Wake",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "abz",
+      "name": "Abui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aca",
+      "name": "Achagua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acb",
+      "name": "Áncá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acd",
+      "name": "Gikyode",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ace",
+      "name": "Achinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acf",
+      "inverted_name": "Creole French, Saint Lucian",
+      "name": "Saint Lucian Creole French",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ach",
+      "name": "Acoli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aci",
+      "name": "Aka-Cari",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ack",
+      "name": "Aka-Kora",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "acl",
+      "name": "Akar-Bale",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "acm",
+      "inverted_name": "Arabic, Mesopotamian",
+      "name": "Mesopotamian Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acn",
+      "name": "Achang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acp",
+      "inverted_name": "Acipa, Eastern",
+      "name": "Eastern Acipa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acq",
+      "inverted_name": "Arabic, Ta'izzi-Adeni",
+      "name": "Ta'izzi-Adeni Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acr",
+      "name": "Achi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acs",
+      "name": "Acroá",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "act",
+      "name": "Achterhoeks",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acu",
+      "name": "Achuar-Shiwiar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acv",
+      "name": "Achumawi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acw",
+      "inverted_name": "Arabic, Hijazi",
+      "name": "Hijazi Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acx",
+      "inverted_name": "Arabic, Omani",
+      "name": "Omani Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acy",
+      "inverted_name": "Arabic, Cypriot",
+      "name": "Cypriot Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "acz",
+      "name": "Acheron",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ada",
+      "name": "Adangme",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adb",
+      "name": "Atauran",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "add",
+      "name": "Lidzonka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ade",
+      "name": "Adele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adf",
+      "inverted_name": "Arabic, Dhofari",
+      "name": "Dhofari Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adg",
+      "name": "Andegerebinha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adh",
+      "name": "Adhola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adi",
+      "name": "Adi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adj",
+      "name": "Adioukrou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adl",
+      "name": "Galo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adn",
+      "name": "Adang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ado",
+      "name": "Abu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adq",
+      "name": "Adangbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adr",
+      "name": "Adonara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ads",
+      "name": "Adamorobe Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adt",
+      "name": "Adnyamathanha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adu",
+      "name": "Aduge",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adw",
+      "name": "Amundava",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adx",
+      "inverted_name": "Tibetan, Amdo",
+      "name": "Amdo Tibetan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ady",
+      "name": "Adyghe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "adz",
+      "name": "Adzera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aea",
+      "name": "Areba",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "aeb",
+      "inverted_name": "Arabic, Tunisian",
+      "name": "Tunisian Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aec",
+      "inverted_name": "Arabic, Saidi",
+      "name": "Saidi Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aed",
+      "name": "Argentine Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aee",
+      "inverted_name": "Pashai, Northeast",
+      "name": "Northeast Pashai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aek",
+      "name": "Haeke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ael",
+      "name": "Ambele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aem",
+      "name": "Arem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aen",
+      "name": "Armenian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aeq",
+      "name": "Aer",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aer",
+      "inverted_name": "Arrernte, Eastern",
+      "name": "Eastern Arrernte",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aes",
+      "name": "Alsea",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "aeu",
+      "name": "Akeu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aew",
+      "name": "Ambakich",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aey",
+      "name": "Amele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aez",
+      "name": "Aeka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "afb",
+      "inverted_name": "Arabic, Gulf",
+      "name": "Gulf Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "afd",
+      "name": "Andai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "afe",
+      "name": "Putukwam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "afg",
+      "name": "Afghan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "afh",
+      "name": "Afrihili",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "afi",
+      "name": "Akrukay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "afk",
+      "name": "Nanubae",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "afn",
+      "name": "Defaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "afo",
+      "name": "Eloyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "afp",
+      "name": "Tapei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "af",
+      "alpha_3": "afr",
+      "name": "Afrikaans",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "afs",
+      "inverted_name": "Creole, Afro-Seminole",
+      "name": "Afro-Seminole Creole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aft",
+      "name": "Afitti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "afu",
+      "name": "Awutu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "afz",
+      "name": "Obokuitai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aga",
+      "name": "Aguano",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "agb",
+      "name": "Legbo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agc",
+      "name": "Agatu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agd",
+      "name": "Agarabi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "age",
+      "name": "Angal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agf",
+      "name": "Arguni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agg",
+      "name": "Angor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agh",
+      "name": "Ngelima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agi",
+      "name": "Agariya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agj",
+      "name": "Argobba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agk",
+      "inverted_name": "Agta, Isarog",
+      "name": "Isarog Agta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agl",
+      "name": "Fembe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agm",
+      "name": "Angaataha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agn",
+      "name": "Agutaynen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ago",
+      "name": "Tainae",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agq",
+      "name": "Aghem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agr",
+      "name": "Aguaruna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ags",
+      "name": "Esimbi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agt",
+      "inverted_name": "Agta, Central Cagayan",
+      "name": "Central Cagayan Agta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agu",
+      "name": "Aguacateco",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agv",
+      "inverted_name": "Dumagat, Remontado",
+      "name": "Remontado Dumagat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agw",
+      "name": "Kahua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agx",
+      "name": "Aghul",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agy",
+      "inverted_name": "Alta, Southern",
+      "name": "Southern Alta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "agz",
+      "inverted_name": "Agta, Mt. Iriga",
+      "name": "Mt. Iriga Agta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aha",
+      "name": "Ahanta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ahb",
+      "name": "Axamb",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ahg",
+      "name": "Qimant",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ahh",
+      "name": "Aghu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ahi",
+      "inverted_name": "Aizi, Tiagbamrin",
+      "name": "Tiagbamrin Aizi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ahk",
+      "name": "Akha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ahl",
+      "name": "Igo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ahm",
+      "inverted_name": "Aizi, Mobumrin",
+      "name": "Mobumrin Aizi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ahn",
+      "name": "Àhàn",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aho",
+      "name": "Ahom",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ahp",
+      "inverted_name": "Aizi, Aproumu",
+      "name": "Aproumu Aizi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ahr",
+      "name": "Ahirani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ahs",
+      "name": "Ashe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aht",
+      "name": "Ahtena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aia",
+      "name": "Arosi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aib",
+      "name": "Ainu (China)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aic",
+      "name": "Ainbai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aid",
+      "name": "Alngith",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "aie",
+      "name": "Amara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aif",
+      "name": "Agi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aig",
+      "inverted_name": "Creole English, Antigua and Barbuda",
+      "name": "Antigua and Barbuda Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aih",
+      "name": "Ai-Cham",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aii",
+      "inverted_name": "Neo-Aramaic, Assyrian",
+      "name": "Assyrian Neo-Aramaic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aij",
+      "name": "Lishanid Noshan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aik",
+      "name": "Ake",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ail",
+      "name": "Aimele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aim",
+      "name": "Aimol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ain",
+      "name": "Ainu (Japan)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aio",
+      "name": "Aiton",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aip",
+      "name": "Burumakok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aiq",
+      "name": "Aimaq",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "air",
+      "name": "Airoran",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ait",
+      "name": "Arikem",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "aiw",
+      "name": "Aari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aix",
+      "name": "Aighon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aiy",
+      "name": "Ali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aja",
+      "name": "Aja (South Sudan)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ajg",
+      "name": "Aja (Benin)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aji",
+      "name": "Ajië",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ajn",
+      "name": "Andajin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ajp",
+      "inverted_name": "Arabic, South Levantine",
+      "name": "South Levantine Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ajs",
+      "name": "Algerian Jewish Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aju",
+      "inverted_name": "Arabic, Judeo-Moroccan",
+      "name": "Judeo-Moroccan Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ajw",
+      "name": "Ajawa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ajz",
+      "inverted_name": "Karbi, Amri",
+      "name": "Amri Karbi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ak",
+      "alpha_3": "aka",
+      "name": "Akan",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akb",
+      "name": "Batak Angkola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akc",
+      "name": "Mpur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akd",
+      "name": "Ukpet-Ehom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ake",
+      "name": "Akawaio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akf",
+      "name": "Akpa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akg",
+      "name": "Anakalangu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akh",
+      "name": "Angal Heneng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aki",
+      "name": "Aiome",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akj",
+      "name": "Aka-Jeru",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "akk",
+      "name": "Akkadian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "akl",
+      "name": "Aklanon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akm",
+      "name": "Aka-Bo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ako",
+      "name": "Akurio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akp",
+      "name": "Siwu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akq",
+      "name": "Ak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akr",
+      "name": "Araki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aks",
+      "name": "Akaselem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akt",
+      "name": "Akolet",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aku",
+      "name": "Akum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akv",
+      "name": "Akhvakh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akw",
+      "name": "Akwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "akx",
+      "name": "Aka-Kede",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "aky",
+      "name": "Aka-Kol",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "akz",
+      "name": "Alabama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ala",
+      "name": "Alago",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alc",
+      "name": "Qawasqar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ald",
+      "name": "Alladian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ale",
+      "name": "Aleut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alf",
+      "name": "Alege",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alh",
+      "name": "Alawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ali",
+      "name": "Amaimon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alj",
+      "name": "Alangan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alk",
+      "name": "Alak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "all",
+      "name": "Allar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alm",
+      "name": "Amblong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aln",
+      "inverted_name": "Albanian, Gheg",
+      "name": "Gheg Albanian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alo",
+      "name": "Larike-Wakasihu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alp",
+      "name": "Alune",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alq",
+      "name": "Algonquin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alr",
+      "name": "Alutor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "als",
+      "inverted_name": "Albanian, Tosk",
+      "name": "Tosk Albanian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alt",
+      "inverted_name": "Altai, Southern",
+      "name": "Southern Altai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alu",
+      "name": "'Are'are",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alw",
+      "name": "Alaba-K’abeena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alx",
+      "name": "Amol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aly",
+      "name": "Alyawarr",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "alz",
+      "name": "Alur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ama",
+      "name": "Amanayé",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "amb",
+      "name": "Ambo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amc",
+      "name": "Amahuaca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ame",
+      "name": "Yanesha'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amf",
+      "name": "Hamer-Banna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amg",
+      "name": "Amurdak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "am",
+      "alpha_3": "amh",
+      "name": "Amharic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ami",
+      "name": "Amis",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amj",
+      "name": "Amdang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amk",
+      "name": "Ambai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aml",
+      "name": "War-Jaintia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amm",
+      "name": "Ama (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amn",
+      "name": "Amanab",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amo",
+      "name": "Amo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amp",
+      "name": "Alamblak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amq",
+      "name": "Amahai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amr",
+      "name": "Amarakaeri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ams",
+      "inverted_name": "Amami-Oshima, Southern",
+      "name": "Southern Amami-Oshima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amt",
+      "name": "Amto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amu",
+      "inverted_name": "Amuzgo, Guerrero",
+      "name": "Guerrero Amuzgo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amv",
+      "name": "Ambelau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amw",
+      "inverted_name": "Neo-Aramaic, Western",
+      "name": "Western Neo-Aramaic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amx",
+      "name": "Anmatyerre",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amy",
+      "name": "Ami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "amz",
+      "name": "Atampaya",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ana",
+      "name": "Andaqui",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "anb",
+      "name": "Andoa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "anc",
+      "name": "Ngas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "and",
+      "name": "Ansus",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ane",
+      "name": "Xârâcùù",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "anf",
+      "name": "Animere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ang",
+      "inverted_name": "English, Old (ca. 450-1100)",
+      "name": "Old English (ca. 450-1100)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "anh",
+      "name": "Nend",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ani",
+      "name": "Andi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "anj",
+      "name": "Anor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ank",
+      "name": "Goemai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "anl",
+      "inverted_name": "Chin, Anu-Hkongso",
+      "name": "Anu-Hkongso Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "anm",
+      "name": "Anal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ann",
+      "name": "Obolo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ano",
+      "name": "Andoque",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "anp",
+      "name": "Angika",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "anq",
+      "name": "Jarawa (India)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "anr",
+      "name": "Andh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ans",
+      "name": "Anserma",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ant",
+      "name": "Antakarinya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "anu",
+      "name": "Anuak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "anv",
+      "name": "Denya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "anw",
+      "name": "Anaang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "anx",
+      "name": "Andra-Hus",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "any",
+      "name": "Anyin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "anz",
+      "name": "Anem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aoa",
+      "name": "Angolar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aob",
+      "name": "Abom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aoc",
+      "name": "Pemon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aod",
+      "name": "Andarum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aoe",
+      "name": "Angal Enen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aof",
+      "name": "Bragat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aog",
+      "name": "Angoram",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aoi",
+      "name": "Anindilyakwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aoj",
+      "name": "Mufian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aok",
+      "name": "Arhö",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aol",
+      "name": "Alor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aom",
+      "name": "Ömie",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aon",
+      "inverted_name": "Arapesh, Bumbita",
+      "name": "Bumbita Arapesh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aor",
+      "name": "Aore",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "aos",
+      "name": "Taikat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aot",
+      "name": "Atong (India)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aou",
+      "name": "A'ou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aox",
+      "name": "Atorada",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aoz",
+      "name": "Uab Meto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apb",
+      "name": "Sa'a",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apc",
+      "inverted_name": "Arabic, North Levantine",
+      "name": "North Levantine Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apd",
+      "inverted_name": "Arabic, Sudanese",
+      "name": "Sudanese Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ape",
+      "name": "Bukiyip",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apf",
+      "inverted_name": "Agta, Pahanan",
+      "name": "Pahanan Agta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apg",
+      "name": "Ampanang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aph",
+      "name": "Athpariya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "api",
+      "name": "Apiaká",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apj",
+      "inverted_name": "Apache, Jicarilla",
+      "name": "Jicarilla Apache",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apk",
+      "inverted_name": "Apache, Kiowa",
+      "name": "Kiowa Apache",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apl",
+      "inverted_name": "Apache, Lipan",
+      "name": "Lipan Apache",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apm",
+      "inverted_name": "Apache, Mescalero-Chiricahua",
+      "name": "Mescalero-Chiricahua Apache",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apn",
+      "name": "Apinayé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apo",
+      "name": "Ambul",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "app",
+      "name": "Apma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apq",
+      "name": "A-Pucikwar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apr",
+      "name": "Arop-Lokep",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aps",
+      "name": "Arop-Sissano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apt",
+      "name": "Apatani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apu",
+      "name": "Apurinã",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apv",
+      "name": "Alapmunte",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "apw",
+      "inverted_name": "Apache, Western",
+      "name": "Western Apache",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apx",
+      "name": "Aputai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apy",
+      "name": "Apalaí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "apz",
+      "name": "Safeyoka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aqc",
+      "name": "Archi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aqd",
+      "inverted_name": "Dogon, Ampari",
+      "name": "Ampari Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aqg",
+      "name": "Arigidi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aqk",
+      "name": "Aninka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aqm",
+      "name": "Atohwaim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aqn",
+      "inverted_name": "Alta, Northern",
+      "name": "Northern Alta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aqp",
+      "name": "Atakapa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "aqr",
+      "name": "Arhâ",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aqt",
+      "name": "Angaité",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aqz",
+      "name": "Akuntsu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ar",
+      "alpha_3": "ara",
+      "name": "Arabic",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "arb",
+      "inverted_name": "Arabic, Standard",
+      "name": "Standard Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "arc",
+      "inverted_name": "Aramaic, Official (700-300 BCE)",
+      "name": "Official Aramaic (700-300 BCE)",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "ard",
+      "name": "Arabana",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "are",
+      "inverted_name": "Arrarnta, Western",
+      "name": "Western Arrarnta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "an",
+      "alpha_3": "arg",
+      "name": "Aragonese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "arh",
+      "name": "Arhuaco",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ari",
+      "name": "Arikara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "arj",
+      "name": "Arapaso",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ark",
+      "name": "Arikapú",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "arl",
+      "name": "Arabela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "arn",
+      "name": "Mapudungun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aro",
+      "name": "Araona",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "arp",
+      "name": "Arapaho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "arq",
+      "inverted_name": "Arabic, Algerian",
+      "name": "Algerian Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "arr",
+      "name": "Karo (Brazil)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ars",
+      "inverted_name": "Arabic, Najdi",
+      "name": "Najdi Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aru",
+      "name": "Aruá (Amazonas State)",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "arv",
+      "name": "Arbore",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "arw",
+      "name": "Arawak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "arx",
+      "name": "Aruá (Rodonia State)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ary",
+      "inverted_name": "Arabic, Moroccan",
+      "name": "Moroccan Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "arz",
+      "inverted_name": "Arabic, Egyptian",
+      "name": "Egyptian Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asa",
+      "name": "Asu (Tanzania)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asb",
+      "name": "Assiniboine",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asc",
+      "inverted_name": "Asmat, Casuarina Coast",
+      "name": "Casuarina Coast Asmat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ase",
+      "name": "American Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asf",
+      "name": "Auslan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asg",
+      "name": "Cishingini",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ash",
+      "name": "Abishira",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "asi",
+      "name": "Buruwai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asj",
+      "name": "Sari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ask",
+      "name": "Ashkun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asl",
+      "name": "Asilulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "as",
+      "alpha_3": "asm",
+      "name": "Assamese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asn",
+      "inverted_name": "Asuriní, Xingú",
+      "name": "Xingú Asuriní",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aso",
+      "name": "Dano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asp",
+      "name": "Algerian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asq",
+      "name": "Austrian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asr",
+      "name": "Asuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ass",
+      "name": "Ipulo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ast",
+      "name": "Asturian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asu",
+      "inverted_name": "Asurini, Tocantins",
+      "name": "Tocantins Asurini",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asv",
+      "name": "Asoa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asw",
+      "name": "Australian Aborigines Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asx",
+      "name": "Muratayak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asy",
+      "inverted_name": "Asmat, Yaosakor",
+      "name": "Yaosakor Asmat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "asz",
+      "name": "As",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ata",
+      "name": "Pele-Ata",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atb",
+      "name": "Zaiwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atc",
+      "name": "Atsahuaca",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "atd",
+      "inverted_name": "Manobo, Ata",
+      "name": "Ata Manobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ate",
+      "name": "Atemble",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atg",
+      "name": "Ivbie North-Okpela-Arhe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ati",
+      "name": "Attié",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atj",
+      "name": "Atikamekw",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atk",
+      "name": "Ati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atl",
+      "inverted_name": "Agta, Mt. Iraya",
+      "name": "Mt. Iraya Agta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atm",
+      "name": "Ata",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atn",
+      "name": "Ashtiani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ato",
+      "name": "Atong (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atp",
+      "inverted_name": "Atta, Pudtol",
+      "name": "Pudtol Atta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atq",
+      "name": "Aralle-Tabulahan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atr",
+      "name": "Waimiri-Atroari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ats",
+      "name": "Gros Ventre",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "att",
+      "inverted_name": "Atta, Pamplona",
+      "name": "Pamplona Atta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atu",
+      "name": "Reel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atv",
+      "inverted_name": "Altai, Northern",
+      "name": "Northern Altai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atw",
+      "name": "Atsugewi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atx",
+      "name": "Arutani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aty",
+      "name": "Aneityum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "atz",
+      "name": "Arta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aua",
+      "name": "Asumboa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aub",
+      "name": "Alugu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "auc",
+      "name": "Waorani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aud",
+      "name": "Anuta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aug",
+      "name": "Aguna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "auh",
+      "name": "Aushi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aui",
+      "name": "Anuki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "auj",
+      "name": "Awjilah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "auk",
+      "name": "Heyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aul",
+      "name": "Aulua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aum",
+      "name": "Asu (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aun",
+      "inverted_name": "One, Molmo",
+      "name": "Molmo One",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "auo",
+      "name": "Auyokawa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "aup",
+      "name": "Makayam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "auq",
+      "name": "Anus",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aur",
+      "name": "Aruek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aut",
+      "name": "Austral",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "auu",
+      "name": "Auye",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "auw",
+      "name": "Awyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aux",
+      "name": "Aurá",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "auy",
+      "name": "Awiyaana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "auz",
+      "inverted_name": "Arabic, Uzbeki",
+      "name": "Uzbeki Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "av",
+      "alpha_3": "ava",
+      "name": "Avaric",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "avb",
+      "name": "Avau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "avd",
+      "name": "Alviri-Vidari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ae",
+      "alpha_3": "ave",
+      "name": "Avestan",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "avi",
+      "name": "Avikam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "avk",
+      "name": "Kotava",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "avl",
+      "inverted_name": "Arabic, Eastern Egyptian Bedawi",
+      "name": "Eastern Egyptian Bedawi Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "avm",
+      "name": "Angkamuthi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "avn",
+      "name": "Avatime",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "avo",
+      "name": "Agavotaguerra",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "avs",
+      "name": "Aushiri",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "avt",
+      "name": "Au",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "avu",
+      "name": "Avokaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "avv",
+      "name": "Avá-Canoeiro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awa",
+      "name": "Awadhi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awb",
+      "name": "Awa (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awc",
+      "name": "Cicipu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awe",
+      "name": "Awetí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awg",
+      "name": "Anguthimri",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "awh",
+      "name": "Awbono",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awi",
+      "name": "Aekyom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awk",
+      "name": "Awabakal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "awm",
+      "name": "Arawum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awn",
+      "name": "Awngi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awo",
+      "name": "Awak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awr",
+      "name": "Awera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aws",
+      "inverted_name": "Awyu, South",
+      "name": "South Awyu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awt",
+      "name": "Araweté",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awu",
+      "inverted_name": "Awyu, Central",
+      "name": "Central Awyu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awv",
+      "inverted_name": "Awyu, Jair",
+      "name": "Jair Awyu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aww",
+      "name": "Awun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awx",
+      "name": "Awara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "awy",
+      "inverted_name": "Awyu, Edera",
+      "name": "Edera Awyu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "axb",
+      "name": "Abipon",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "axe",
+      "name": "Ayerrerenge",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "axg",
+      "inverted_name": "Arára, Mato Grosso",
+      "name": "Mato Grosso Arára",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "axk",
+      "name": "Yaka (Central African Republic)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "axl",
+      "inverted_name": "Aranda, Lower Southern",
+      "name": "Lower Southern Aranda",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "axm",
+      "inverted_name": "Armenian, Middle",
+      "name": "Middle Armenian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "axx",
+      "name": "Xârâgurè",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aya",
+      "name": "Awar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayb",
+      "inverted_name": "Gbe, Ayizo",
+      "name": "Ayizo Gbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayc",
+      "inverted_name": "Aymara, Southern",
+      "name": "Southern Aymara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayd",
+      "name": "Ayabadhu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "aye",
+      "name": "Ayere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayg",
+      "name": "Ginyanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayh",
+      "inverted_name": "Arabic, Hadrami",
+      "name": "Hadrami Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayi",
+      "name": "Leyigha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayk",
+      "name": "Akuku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayl",
+      "inverted_name": "Arabic, Libyan",
+      "name": "Libyan Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ay",
+      "alpha_3": "aym",
+      "name": "Aymara",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayn",
+      "inverted_name": "Arabic, Sanaani",
+      "name": "Sanaani Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayo",
+      "name": "Ayoreo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayp",
+      "inverted_name": "Arabic, North Mesopotamian",
+      "name": "North Mesopotamian Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayq",
+      "name": "Ayi (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayr",
+      "inverted_name": "Aymara, Central",
+      "name": "Central Aymara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ays",
+      "inverted_name": "Ayta, Sorsogon",
+      "name": "Sorsogon Ayta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayt",
+      "inverted_name": "Ayta, Magbukun",
+      "name": "Magbukun Ayta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayu",
+      "name": "Ayu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ayz",
+      "name": "Mai Brat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "aza",
+      "name": "Azha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "azb",
+      "inverted_name": "Azerbaijani, South",
+      "name": "South Azerbaijani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "azd",
+      "inverted_name": "Nahuatl, Eastern Durango",
+      "name": "Eastern Durango Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "az",
+      "alpha_3": "aze",
+      "name": "Azerbaijani",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "azg",
+      "inverted_name": "Amuzgo, San Pedro Amuzgos",
+      "name": "San Pedro Amuzgos Amuzgo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "azj",
+      "inverted_name": "Azerbaijani, North",
+      "name": "North Azerbaijani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "azm",
+      "inverted_name": "Amuzgo, Ipalapa",
+      "name": "Ipalapa Amuzgo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "azn",
+      "inverted_name": "Nahuatl, Western Durango",
+      "name": "Western Durango Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "azo",
+      "name": "Awing",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "azt",
+      "inverted_name": "Atta, Faire",
+      "name": "Faire Atta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "azz",
+      "inverted_name": "Nahuatl, Highland Puebla",
+      "name": "Highland Puebla Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "baa",
+      "name": "Babatana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bab",
+      "name": "Bainouk-Gunyuño",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bac",
+      "name": "Badui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bae",
+      "name": "Baré",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "baf",
+      "name": "Nubaca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bag",
+      "name": "Tuki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bah",
+      "inverted_name": "Creole English, Bahamas",
+      "name": "Bahamas Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "baj",
+      "name": "Barakai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ba",
+      "alpha_3": "bak",
+      "name": "Bashkir",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bal",
+      "name": "Baluchi",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_2": "bm",
+      "alpha_3": "bam",
+      "name": "Bambara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ban",
+      "name": "Balinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bao",
+      "name": "Waimaha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bap",
+      "name": "Bantawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bar",
+      "name": "Bavarian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bas",
+      "name": "Basa (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bau",
+      "name": "Bada (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bav",
+      "name": "Vengo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "baw",
+      "name": "Bambili-Bambui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bax",
+      "name": "Bamun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bay",
+      "name": "Batuley",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bba",
+      "name": "Baatonum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbb",
+      "name": "Barai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbc",
+      "name": "Batak Toba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbd",
+      "name": "Bau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbe",
+      "name": "Bangba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbf",
+      "name": "Baibai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbg",
+      "name": "Barama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbh",
+      "name": "Bugan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbi",
+      "name": "Barombi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbj",
+      "name": "Ghomálá'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbk",
+      "name": "Babanki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbl",
+      "name": "Bats",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbm",
+      "name": "Babango",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbn",
+      "name": "Uneapa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbo",
+      "inverted_name": "Bobo Madaré, Northern",
+      "name": "Northern Bobo Madaré",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbp",
+      "inverted_name": "Banda, West Central",
+      "name": "West Central Banda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbq",
+      "name": "Bamali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbr",
+      "name": "Girawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbs",
+      "name": "Bakpinka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbt",
+      "name": "Mburku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbu",
+      "name": "Kulung (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbv",
+      "name": "Karnai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbw",
+      "name": "Baba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bbx",
+      "name": "Bubia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bby",
+      "name": "Befang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bca",
+      "inverted_name": "Bai, Central",
+      "name": "Central Bai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcb",
+      "name": "Bainouk-Samik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcc",
+      "inverted_name": "Balochi, Southern",
+      "name": "Southern Balochi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcd",
+      "inverted_name": "Babar, North",
+      "name": "North Babar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bce",
+      "name": "Bamenyam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcf",
+      "name": "Bamu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcg",
+      "name": "Baga Pokur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bch",
+      "name": "Bariai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bci",
+      "name": "Baoulé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcj",
+      "name": "Bardi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bck",
+      "name": "Bunuba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcl",
+      "inverted_name": "Bikol, Central",
+      "name": "Central Bikol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcm",
+      "name": "Bannoni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcn",
+      "name": "Bali (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bco",
+      "name": "Kaluli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcp",
+      "name": "Bali (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcq",
+      "name": "Bench",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcr",
+      "name": "Babine",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcs",
+      "name": "Kohumono",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bct",
+      "name": "Bendi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcu",
+      "name": "Awad Bing",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcv",
+      "name": "Shoo-Minda-Nye",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcw",
+      "name": "Bana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcy",
+      "name": "Bacama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bcz",
+      "name": "Bainouk-Gunyaamolo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bda",
+      "name": "Bayot",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdb",
+      "name": "Basap",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdc",
+      "name": "Emberá-Baudó",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdd",
+      "name": "Bunama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bde",
+      "name": "Bade",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdf",
+      "name": "Biage",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdg",
+      "name": "Bonggi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdh",
+      "name": "Baka (South Sudan)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdi",
+      "name": "Burun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdj",
+      "name": "Bai (South Sudan)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdk",
+      "name": "Budukh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdl",
+      "inverted_name": "Bajau, Indonesian",
+      "name": "Indonesian Bajau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdm",
+      "name": "Buduma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdn",
+      "name": "Baldemu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdo",
+      "name": "Morom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdp",
+      "name": "Bende",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdq",
+      "name": "Bahnar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdr",
+      "inverted_name": "Bajau, West Coast",
+      "name": "West Coast Bajau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bds",
+      "name": "Burunge",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdt",
+      "name": "Bokoto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdu",
+      "name": "Oroko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdv",
+      "name": "Bodo Parja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdw",
+      "name": "Baham",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdx",
+      "name": "Budong-Budong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdy",
+      "name": "Bandjalang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bdz",
+      "name": "Badeshi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bea",
+      "name": "Beaver",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "beb",
+      "name": "Bebele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bec",
+      "name": "Iceve-Maci",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bed",
+      "name": "Bedoanas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bee",
+      "name": "Byangsi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bef",
+      "name": "Benabena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "beg",
+      "name": "Belait",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "beh",
+      "name": "Biali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bei",
+      "name": "Bekati'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bej",
+      "name": "Beja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bek",
+      "name": "Bebeli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "be",
+      "alpha_3": "bel",
+      "name": "Belarusian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bem",
+      "name": "Bemba (Zambia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "bn",
+      "alpha_3": "ben",
+      "common_name": "Bangla",
+      "name": "Bengali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "beo",
+      "name": "Beami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bep",
+      "name": "Besoa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "beq",
+      "name": "Beembe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bes",
+      "name": "Besme",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bet",
+      "inverted_name": "Béte, Guiberoua",
+      "name": "Guiberoua Béte",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "beu",
+      "name": "Blagar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bev",
+      "inverted_name": "Bété, Daloa",
+      "name": "Daloa Bété",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bew",
+      "name": "Betawi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bex",
+      "name": "Jur Modo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bey",
+      "name": "Beli (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bez",
+      "name": "Bena (Tanzania)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfa",
+      "name": "Bari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfb",
+      "inverted_name": "Bareli, Pauri",
+      "name": "Pauri Bareli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfc",
+      "inverted_name": "Bai, Panyi",
+      "name": "Panyi Bai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfd",
+      "name": "Bafut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfe",
+      "name": "Betaf",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bff",
+      "name": "Bofi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfg",
+      "inverted_name": "Kayan, Busang",
+      "name": "Busang Kayan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfh",
+      "name": "Blafe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfi",
+      "name": "British Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfj",
+      "name": "Bafanji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfk",
+      "name": "Ban Khor Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfl",
+      "name": "Banda-Ndélé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfm",
+      "name": "Mmen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfn",
+      "name": "Bunak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfo",
+      "inverted_name": "Birifor, Malba",
+      "name": "Malba Birifor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfp",
+      "name": "Beba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfq",
+      "name": "Badaga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfr",
+      "name": "Bazigar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfs",
+      "inverted_name": "Bai, Southern",
+      "name": "Southern Bai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bft",
+      "name": "Balti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfu",
+      "name": "Gahri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfw",
+      "name": "Bondo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfx",
+      "name": "Bantayanon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfy",
+      "name": "Bagheli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bfz",
+      "inverted_name": "Pahari, Mahasu",
+      "name": "Mahasu Pahari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bga",
+      "name": "Gwamhi-Wuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgb",
+      "name": "Bobongko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgc",
+      "name": "Haryanvi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgd",
+      "inverted_name": "Bareli, Rathwi",
+      "name": "Rathwi Bareli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bge",
+      "name": "Bauria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgf",
+      "name": "Bangandu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgg",
+      "name": "Bugun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgi",
+      "name": "Giangan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgj",
+      "name": "Bangolan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgk",
+      "name": "Bit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgl",
+      "name": "Bo (Laos)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgn",
+      "inverted_name": "Balochi, Western",
+      "name": "Western Balochi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgo",
+      "name": "Baga Koga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgp",
+      "inverted_name": "Balochi, Eastern",
+      "name": "Eastern Balochi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgq",
+      "name": "Bagri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgr",
+      "inverted_name": "Chin, Bawm",
+      "name": "Bawm Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgs",
+      "name": "Tagabawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgt",
+      "name": "Bughotu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgu",
+      "name": "Mbongno",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgv",
+      "name": "Warkay-Bipim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgw",
+      "name": "Bhatri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgx",
+      "inverted_name": "Turkish, Balkan Gagauz",
+      "name": "Balkan Gagauz Turkish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgy",
+      "name": "Benggoi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bgz",
+      "name": "Banggai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bha",
+      "name": "Bharia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhb",
+      "name": "Bhili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhc",
+      "name": "Biga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhd",
+      "name": "Bhadrawahi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhe",
+      "name": "Bhaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhf",
+      "name": "Odiai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhg",
+      "name": "Binandere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhh",
+      "name": "Bukharic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhi",
+      "name": "Bhilali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhj",
+      "name": "Bahing",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhl",
+      "name": "Bimin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhm",
+      "name": "Bathari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhn",
+      "inverted_name": "Neo-Aramaic, Bohtan",
+      "name": "Bohtan Neo-Aramaic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bho",
+      "name": "Bhojpuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhp",
+      "name": "Bima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhq",
+      "name": "Tukang Besi South",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhr",
+      "inverted_name": "Malagasy, Bara",
+      "name": "Bara Malagasy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhs",
+      "name": "Buwal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bht",
+      "name": "Bhattiyali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhu",
+      "name": "Bhunjia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhv",
+      "name": "Bahau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhw",
+      "name": "Biak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhx",
+      "name": "Bhalay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhy",
+      "name": "Bhele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bhz",
+      "name": "Bada (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bia",
+      "name": "Badimaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bib",
+      "name": "Bissa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bid",
+      "name": "Bidiyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bie",
+      "name": "Bepour",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bif",
+      "name": "Biafada",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "big",
+      "name": "Biangai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bik",
+      "name": "Bikol",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bil",
+      "name": "Bile",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bim",
+      "name": "Bimoba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bin",
+      "name": "Bini",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bio",
+      "name": "Nai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bip",
+      "name": "Bila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "biq",
+      "name": "Bipi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bir",
+      "name": "Bisorio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "bi",
+      "alpha_3": "bis",
+      "name": "Bislama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bit",
+      "name": "Berinomo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "biu",
+      "name": "Biete",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "biv",
+      "inverted_name": "Birifor, Southern",
+      "name": "Southern Birifor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "biw",
+      "name": "Kol (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bix",
+      "name": "Bijori",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "biy",
+      "name": "Birhor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "biz",
+      "name": "Baloi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bja",
+      "name": "Budza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjb",
+      "name": "Banggarla",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "bjc",
+      "name": "Bariji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bje",
+      "inverted_name": "Mien, Biao-Jiao",
+      "name": "Biao-Jiao Mien",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjf",
+      "inverted_name": "Neo-Aramaic, Barzani Jewish",
+      "name": "Barzani Jewish Neo-Aramaic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjg",
+      "name": "Bidyogo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjh",
+      "name": "Bahinemo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bji",
+      "name": "Burji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjj",
+      "name": "Kanauji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjk",
+      "name": "Barok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjl",
+      "name": "Bulu (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjm",
+      "name": "Bajelani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjn",
+      "name": "Banjar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjo",
+      "inverted_name": "Banda, Mid-Southern",
+      "name": "Mid-Southern Banda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjp",
+      "name": "Fanamaket",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjr",
+      "name": "Binumarien",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjs",
+      "name": "Bajan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjt",
+      "name": "Balanta-Ganja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bju",
+      "name": "Busuu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjv",
+      "name": "Bedjond",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjw",
+      "name": "Bakwé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjx",
+      "inverted_name": "Itneg, Banao",
+      "name": "Banao Itneg",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bjy",
+      "name": "Bayali",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "bjz",
+      "name": "Baruga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bka",
+      "name": "Kyak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkc",
+      "name": "Baka (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkd",
+      "name": "Binukid",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkf",
+      "name": "Beeke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkg",
+      "name": "Buraka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkh",
+      "name": "Bakoko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bki",
+      "name": "Baki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkj",
+      "name": "Pande",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkk",
+      "name": "Brokskat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkl",
+      "name": "Berik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkm",
+      "name": "Kom (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkn",
+      "name": "Bukitan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bko",
+      "name": "Kwa'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkp",
+      "name": "Boko (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkq",
+      "name": "Bakairí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkr",
+      "name": "Bakumpai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bks",
+      "inverted_name": "Sorsoganon, Northern",
+      "name": "Northern Sorsoganon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkt",
+      "name": "Boloki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bku",
+      "name": "Buhid",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkv",
+      "name": "Bekwarra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkw",
+      "name": "Bekwel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkx",
+      "name": "Baikeno",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bky",
+      "name": "Bokyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bkz",
+      "name": "Bungku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bla",
+      "name": "Siksika",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blb",
+      "name": "Bilua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blc",
+      "name": "Bella Coola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bld",
+      "name": "Bolango",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ble",
+      "name": "Balanta-Kentohe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blf",
+      "name": "Buol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blh",
+      "name": "Kuwaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bli",
+      "name": "Bolia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blj",
+      "name": "Bolongan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blk",
+      "inverted_name": "Karen, Pa'o",
+      "name": "Pa'o Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bll",
+      "name": "Biloxi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "blm",
+      "name": "Beli (South Sudan)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bln",
+      "inverted_name": "Bikol, Southern Catanduanes",
+      "name": "Southern Catanduanes Bikol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blo",
+      "name": "Anii",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blp",
+      "name": "Blablanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blq",
+      "name": "Baluan-Pam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blr",
+      "name": "Blang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bls",
+      "name": "Balaesang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blt",
+      "name": "Tai Dam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blv",
+      "name": "Kibala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blw",
+      "name": "Balangao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blx",
+      "inverted_name": "Ayta, Mag-Indi",
+      "name": "Mag-Indi Ayta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bly",
+      "name": "Notre",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "blz",
+      "name": "Balantak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bma",
+      "name": "Lame",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmb",
+      "name": "Bembe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmc",
+      "name": "Biem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmd",
+      "inverted_name": "Manduri, Baga",
+      "name": "Baga Manduri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bme",
+      "name": "Limassa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmf",
+      "name": "Bom-Kim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmg",
+      "name": "Bamwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmh",
+      "name": "Kein",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmi",
+      "name": "Bagirmi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmj",
+      "name": "Bote-Majhi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmk",
+      "name": "Ghayavi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bml",
+      "name": "Bomboli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmm",
+      "inverted_name": "Malagasy, Northern Betsimisaraka",
+      "name": "Northern Betsimisaraka Malagasy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmn",
+      "name": "Bina (Papua New Guinea)",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "bmo",
+      "name": "Bambalang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmp",
+      "name": "Bulgebi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmq",
+      "name": "Bomu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmr",
+      "name": "Muinane",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bms",
+      "inverted_name": "Kanuri, Bilma",
+      "name": "Bilma Kanuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmt",
+      "name": "Biao Mon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmu",
+      "name": "Somba-Siawari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmv",
+      "name": "Bum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmw",
+      "name": "Bomwali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmx",
+      "name": "Baimak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bmz",
+      "name": "Baramu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bna",
+      "name": "Bonerate",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnb",
+      "name": "Bookan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnc",
+      "name": "Bontok",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnd",
+      "name": "Banda (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bne",
+      "name": "Bintauna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnf",
+      "name": "Masiwang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bng",
+      "name": "Benga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bni",
+      "name": "Bangi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnj",
+      "inverted_name": "Tawbuid, Eastern",
+      "name": "Eastern Tawbuid",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnk",
+      "name": "Bierebo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnl",
+      "name": "Boon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnm",
+      "name": "Batanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnn",
+      "name": "Bunun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bno",
+      "name": "Bantoanon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnp",
+      "name": "Bola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnq",
+      "name": "Bantik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnr",
+      "name": "Butmas-Tur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bns",
+      "name": "Bundeli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnu",
+      "name": "Bentong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnv",
+      "name": "Bonerif",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnw",
+      "name": "Bisis",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnx",
+      "name": "Bangubangu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bny",
+      "name": "Bintulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bnz",
+      "name": "Beezen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "boa",
+      "name": "Bora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bob",
+      "name": "Aweer",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "bo",
+      "alpha_3": "bod",
+      "bibliographic": "tib",
+      "name": "Tibetan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "boe",
+      "name": "Mundabli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bof",
+      "name": "Bolon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bog",
+      "name": "Bamako Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "boh",
+      "name": "Boma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "boi",
+      "name": "Barbareño",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "boj",
+      "name": "Anjam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bok",
+      "name": "Bonjo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bol",
+      "name": "Bole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bom",
+      "name": "Berom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bon",
+      "name": "Bine",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "boo",
+      "inverted_name": "Bozo, Tiemacèwè",
+      "name": "Tiemacèwè Bozo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bop",
+      "name": "Bonkiman",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "boq",
+      "name": "Bogaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bor",
+      "name": "Borôro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "bs",
+      "alpha_3": "bos",
+      "name": "Bosnian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bot",
+      "name": "Bongo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bou",
+      "name": "Bondei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bov",
+      "name": "Tuwuli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bow",
+      "name": "Rema",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "box",
+      "name": "Buamu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "boy",
+      "name": "Bodo (Central African Republic)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "boz",
+      "inverted_name": "Bozo, Tiéyaxo",
+      "name": "Tiéyaxo Bozo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpa",
+      "name": "Daakaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpc",
+      "name": "Mbuk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpd",
+      "name": "Banda-Banda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpe",
+      "name": "Bauni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpg",
+      "name": "Bonggo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bph",
+      "name": "Botlikh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpi",
+      "name": "Bagupi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpj",
+      "name": "Binji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpk",
+      "name": "Orowe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpl",
+      "name": "Broome Pearling Lugger Pidgin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpm",
+      "name": "Biyom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpn",
+      "name": "Dzao Min",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpo",
+      "name": "Anasi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpp",
+      "name": "Kaure",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpq",
+      "inverted_name": "Malay, Banda",
+      "name": "Banda Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpr",
+      "inverted_name": "Blaan, Koronadal",
+      "name": "Koronadal Blaan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bps",
+      "inverted_name": "Blaan, Sarangani",
+      "name": "Sarangani Blaan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpt",
+      "name": "Barrow Point",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "bpu",
+      "name": "Bongu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpv",
+      "inverted_name": "Marind, Bian",
+      "name": "Bian Marind",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpw",
+      "name": "Bo (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpx",
+      "inverted_name": "Bareli, Palya",
+      "name": "Palya Bareli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpy",
+      "name": "Bishnupriya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bpz",
+      "name": "Bilba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqa",
+      "name": "Tchumbuli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqb",
+      "name": "Bagusa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqc",
+      "name": "Boko (Benin)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqd",
+      "name": "Bung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqf",
+      "name": "Baga Kaloum",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "bqg",
+      "name": "Bago-Kusuntu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqh",
+      "name": "Baima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqi",
+      "name": "Bakhtiari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqj",
+      "name": "Bandial",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqk",
+      "name": "Banda-Mbrès",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bql",
+      "name": "Bilakura",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqm",
+      "name": "Wumboko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqn",
+      "name": "Bulgarian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqo",
+      "name": "Balo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqp",
+      "name": "Busa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqq",
+      "name": "Biritai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqr",
+      "name": "Burusu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqs",
+      "name": "Bosngun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqt",
+      "name": "Bamukumbit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqu",
+      "name": "Boguru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqv",
+      "name": "Koro Wachi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqw",
+      "name": "Buru (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqx",
+      "name": "Baangi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqy",
+      "name": "Bengkala Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bqz",
+      "name": "Bakaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bra",
+      "name": "Braj",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brb",
+      "name": "Brao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brc",
+      "inverted_name": "Creole Dutch, Berbice",
+      "name": "Berbice Creole Dutch",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "brd",
+      "name": "Baraamu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "br",
+      "alpha_3": "bre",
+      "name": "Breton",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brf",
+      "name": "Bira",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brg",
+      "name": "Baure",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brh",
+      "name": "Brahui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bri",
+      "name": "Mokpwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brj",
+      "name": "Bieria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brk",
+      "name": "Birked",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "brl",
+      "name": "Birwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brm",
+      "name": "Barambu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brn",
+      "name": "Boruca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bro",
+      "name": "Brokkat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brp",
+      "name": "Barapasi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brq",
+      "name": "Breri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brr",
+      "name": "Birao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brs",
+      "name": "Baras",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brt",
+      "name": "Bitare",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bru",
+      "inverted_name": "Bru, Eastern",
+      "name": "Eastern Bru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brv",
+      "inverted_name": "Bru, Western",
+      "name": "Western Bru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brw",
+      "name": "Bellari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brx",
+      "name": "Bodo (India)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bry",
+      "name": "Burui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "brz",
+      "name": "Bilbil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsa",
+      "name": "Abinomn",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsb",
+      "inverted_name": "Bisaya, Brunei",
+      "name": "Brunei Bisaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsc",
+      "name": "Bassari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bse",
+      "name": "Wushi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsf",
+      "name": "Bauchi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsg",
+      "name": "Bashkardi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsh",
+      "name": "Kati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsi",
+      "name": "Bassossi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsj",
+      "name": "Bangwinji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsk",
+      "name": "Burushaski",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsl",
+      "name": "Basa-Gumna",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "bsm",
+      "name": "Busami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsn",
+      "name": "Barasana-Eduria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bso",
+      "name": "Buso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsp",
+      "name": "Baga Sitemu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsq",
+      "name": "Bassa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsr",
+      "name": "Bassa-Kontagora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bss",
+      "name": "Akoose",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bst",
+      "name": "Basketo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsu",
+      "name": "Bahonsuai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsv",
+      "name": "Baga Sobané",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "bsw",
+      "name": "Baiso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsx",
+      "name": "Yangkam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bsy",
+      "inverted_name": "Bisaya, Sabah",
+      "name": "Sabah Bisaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bta",
+      "name": "Bata",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btc",
+      "name": "Bati (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btd",
+      "name": "Batak Dairi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bte",
+      "name": "Gamo-Ningi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "btf",
+      "name": "Birgit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btg",
+      "inverted_name": "Bété, Gagnoa",
+      "name": "Gagnoa Bété",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bth",
+      "inverted_name": "Bidayuh, Biatah",
+      "name": "Biatah Bidayuh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bti",
+      "name": "Burate",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btj",
+      "inverted_name": "Malay, Bacanese",
+      "name": "Bacanese Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btm",
+      "name": "Batak Mandailing",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btn",
+      "name": "Ratagnon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bto",
+      "inverted_name": "Bikol, Rinconada",
+      "name": "Rinconada Bikol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btp",
+      "name": "Budibud",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btq",
+      "name": "Batek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btr",
+      "name": "Baetora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bts",
+      "name": "Batak Simalungun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btt",
+      "name": "Bete-Bendi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btu",
+      "name": "Batu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btv",
+      "name": "Bateri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btw",
+      "name": "Butuanon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btx",
+      "name": "Batak Karo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bty",
+      "name": "Bobot",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "btz",
+      "name": "Batak Alas-Kluet",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bua",
+      "name": "Buriat",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bub",
+      "name": "Bua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "buc",
+      "name": "Bushi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bud",
+      "name": "Ntcham",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bue",
+      "name": "Beothuk",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "buf",
+      "name": "Bushoong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bug",
+      "name": "Buginese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "buh",
+      "inverted_name": "Bunu, Younuo",
+      "name": "Younuo Bunu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bui",
+      "name": "Bongili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "buj",
+      "name": "Basa-Gurmana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "buk",
+      "name": "Bugawac",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "bg",
+      "alpha_3": "bul",
+      "name": "Bulgarian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bum",
+      "name": "Bulu (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bun",
+      "name": "Sherbro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "buo",
+      "name": "Terei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bup",
+      "name": "Busoa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "buq",
+      "name": "Brem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bus",
+      "name": "Bokobaru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "but",
+      "name": "Bungain",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "buu",
+      "name": "Budu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "buv",
+      "name": "Bun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "buw",
+      "name": "Bubi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bux",
+      "name": "Boghom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "buy",
+      "name": "Bullom So",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "buz",
+      "name": "Bukwen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bva",
+      "name": "Barein",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvb",
+      "name": "Bube",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvc",
+      "name": "Baelelea",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvd",
+      "name": "Baeggu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bve",
+      "inverted_name": "Malay, Berau",
+      "name": "Berau Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvf",
+      "name": "Boor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvg",
+      "name": "Bonkeng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvh",
+      "name": "Bure",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvi",
+      "name": "Belanda Viri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvj",
+      "name": "Baan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvk",
+      "name": "Bukat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvl",
+      "name": "Bolivian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvm",
+      "name": "Bamunka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvn",
+      "name": "Buna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvo",
+      "name": "Bolgo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvp",
+      "name": "Bumang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvq",
+      "name": "Birri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvr",
+      "name": "Burarra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvt",
+      "name": "Bati (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvu",
+      "inverted_name": "Malay, Bukit",
+      "name": "Bukit Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvv",
+      "name": "Baniva",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "bvw",
+      "name": "Boga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvx",
+      "name": "Dibole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvy",
+      "name": "Baybayanon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bvz",
+      "name": "Bauzi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwa",
+      "name": "Bwatoo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwb",
+      "name": "Namosi-Naitasiri-Serua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwc",
+      "name": "Bwile",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwd",
+      "name": "Bwaidoka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwe",
+      "inverted_name": "Karen, Bwe",
+      "name": "Bwe Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwf",
+      "name": "Boselewa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwg",
+      "name": "Barwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwh",
+      "name": "Bishuo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwi",
+      "name": "Baniwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwj",
+      "inverted_name": "Bwamu, Láá Láá",
+      "name": "Láá Láá Bwamu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwk",
+      "name": "Bauwaki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwl",
+      "name": "Bwela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwm",
+      "name": "Biwat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwn",
+      "inverted_name": "Bunu, Wunai",
+      "name": "Wunai Bunu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwo",
+      "name": "Boro (Ethiopia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwp",
+      "name": "Mandobo Bawah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwq",
+      "inverted_name": "Bobo Madaré, Southern",
+      "name": "Southern Bobo Madaré",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwr",
+      "name": "Bura-Pabir",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bws",
+      "name": "Bomboma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwt",
+      "name": "Bafaw-Balong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwu",
+      "name": "Buli (Ghana)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bww",
+      "name": "Bwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwx",
+      "inverted_name": "Bunu, Bu-Nao",
+      "name": "Bu-Nao Bunu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwy",
+      "inverted_name": "Bwamu, Cwi",
+      "name": "Cwi Bwamu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bwz",
+      "name": "Bwisi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxa",
+      "name": "Tairaha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxb",
+      "inverted_name": "Bor, Belanda",
+      "name": "Belanda Bor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxc",
+      "name": "Molengue",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxd",
+      "name": "Pela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxe",
+      "name": "Birale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxf",
+      "name": "Bilur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxg",
+      "name": "Bangala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxh",
+      "name": "Buhutu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxi",
+      "name": "Pirlatapa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "bxj",
+      "name": "Bayungu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxk",
+      "name": "Bukusu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxl",
+      "name": "Jalkunan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxm",
+      "inverted_name": "Buriat, Mongolia",
+      "name": "Mongolia Buriat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxn",
+      "name": "Burduna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxo",
+      "name": "Barikanchi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxp",
+      "name": "Bebil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxq",
+      "name": "Beele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxr",
+      "inverted_name": "Buriat, Russia",
+      "name": "Russia Buriat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxs",
+      "name": "Busam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxu",
+      "inverted_name": "Buriat, China",
+      "name": "China Buriat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxv",
+      "name": "Berakou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxw",
+      "name": "Bankagooma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bxz",
+      "name": "Binahari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bya",
+      "name": "Batak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byb",
+      "name": "Bikya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byc",
+      "name": "Ubaghara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byd",
+      "name": "Benyadu'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bye",
+      "name": "Pouye",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byf",
+      "name": "Bete",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byg",
+      "name": "Baygo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "byh",
+      "name": "Bhujel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byi",
+      "name": "Buyu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byj",
+      "name": "Bina (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byk",
+      "name": "Biao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byl",
+      "name": "Bayono",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bym",
+      "name": "Bidjara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byn",
+      "name": "Bilin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byo",
+      "name": "Biyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byp",
+      "name": "Bumaji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byq",
+      "name": "Basay",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "byr",
+      "name": "Baruya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bys",
+      "name": "Burak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byt",
+      "name": "Berti",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "byv",
+      "name": "Medumba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byw",
+      "name": "Belhariya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byx",
+      "name": "Qaqet",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "byz",
+      "name": "Banaro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bza",
+      "name": "Bandi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzb",
+      "name": "Andio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzc",
+      "inverted_name": "Malagasy, Southern Betsimisaraka",
+      "name": "Southern Betsimisaraka Malagasy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzd",
+      "name": "Bribri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bze",
+      "inverted_name": "Bozo, Jenaama",
+      "name": "Jenaama Bozo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzf",
+      "name": "Boikin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzg",
+      "name": "Babuza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzh",
+      "inverted_name": "Buang, Mapos",
+      "name": "Mapos Buang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzi",
+      "name": "Bisu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzj",
+      "inverted_name": "Kriol English, Belize",
+      "name": "Belize Kriol English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzk",
+      "inverted_name": "Creole English, Nicaragua",
+      "name": "Nicaragua Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzl",
+      "name": "Boano (Sulawesi)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzm",
+      "name": "Bolondo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzn",
+      "name": "Boano (Maluku)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzo",
+      "name": "Bozaba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzp",
+      "name": "Kemberano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzq",
+      "name": "Buli (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzr",
+      "name": "Biri",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "bzs",
+      "name": "Brazilian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzt",
+      "name": "Brithenig",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "bzu",
+      "name": "Burmeso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzv",
+      "name": "Naami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzw",
+      "name": "Basa (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzx",
+      "inverted_name": "Bozo, Kɛlɛngaxo",
+      "name": "Kɛlɛngaxo Bozo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzy",
+      "name": "Obanliku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "bzz",
+      "name": "Evant",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "caa",
+      "name": "Chortí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cab",
+      "name": "Garifuna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cac",
+      "name": "Chuj",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cad",
+      "name": "Caddo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cae",
+      "name": "Lehar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "caf",
+      "inverted_name": "Carrier, Southern",
+      "name": "Southern Carrier",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cag",
+      "name": "Nivaclé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cah",
+      "name": "Cahuarano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "caj",
+      "name": "Chané",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "cak",
+      "name": "Kaqchikel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cal",
+      "name": "Carolinian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cam",
+      "name": "Cemuhî",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "can",
+      "name": "Chambri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cao",
+      "name": "Chácobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cap",
+      "name": "Chipaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "caq",
+      "inverted_name": "Nicobarese, Car",
+      "name": "Car Nicobarese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "car",
+      "inverted_name": "Carib, Galibi",
+      "name": "Galibi Carib",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cas",
+      "name": "Tsimané",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ca",
+      "alpha_3": "cat",
+      "name": "Catalan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cav",
+      "name": "Cavineña",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "caw",
+      "name": "Callawalla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cax",
+      "name": "Chiquitano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cay",
+      "name": "Cayuga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "caz",
+      "name": "Canichana",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "cbb",
+      "name": "Cabiyarí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbc",
+      "name": "Carapana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbd",
+      "name": "Carijona",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbg",
+      "name": "Chimila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbi",
+      "name": "Chachi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbj",
+      "name": "Ede Cabe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbk",
+      "name": "Chavacano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbl",
+      "inverted_name": "Chin, Bualkhaw",
+      "name": "Bualkhaw Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbn",
+      "name": "Nyahkur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbo",
+      "name": "Izora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbq",
+      "name": "Tsucuba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbr",
+      "name": "Cashibo-Cacataibo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbs",
+      "name": "Cashinahua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbt",
+      "name": "Chayahuita",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbu",
+      "name": "Candoshi-Shapra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbv",
+      "name": "Cacua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cbw",
+      "name": "Kinabalian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cby",
+      "name": "Carabayo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ccc",
+      "name": "Chamicuro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ccd",
+      "inverted_name": "Creole, Cafundo",
+      "name": "Cafundo Creole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cce",
+      "name": "Chopi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ccg",
+      "inverted_name": "Daka, Samba",
+      "name": "Samba Daka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cch",
+      "name": "Atsam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ccj",
+      "name": "Kasanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ccl",
+      "name": "Cutchi-Swahili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ccm",
+      "inverted_name": "Creole Malay, Malaccan",
+      "name": "Malaccan Creole Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cco",
+      "inverted_name": "Chinantec, Comaltepec",
+      "name": "Comaltepec Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ccp",
+      "name": "Chakma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ccr",
+      "name": "Cacaopera",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "cda",
+      "name": "Choni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cde",
+      "name": "Chenchu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cdf",
+      "name": "Chiru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cdh",
+      "name": "Chambeali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cdi",
+      "name": "Chodri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cdj",
+      "name": "Churahi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cdm",
+      "name": "Chepang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cdn",
+      "name": "Chaudangsi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cdo",
+      "inverted_name": "Chinese, Min Dong",
+      "name": "Min Dong Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cdr",
+      "name": "Cinda-Regi-Tiyal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cds",
+      "name": "Chadian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cdy",
+      "name": "Chadong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cdz",
+      "name": "Koda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cea",
+      "inverted_name": "Chehalis, Lower",
+      "name": "Lower Chehalis",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ceb",
+      "name": "Cebuano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ceg",
+      "name": "Chamacoco",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cek",
+      "inverted_name": "Chin, Eastern Khumi",
+      "name": "Eastern Khumi Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cen",
+      "name": "Cen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "cs",
+      "alpha_3": "ces",
+      "bibliographic": "cze",
+      "name": "Czech",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cet",
+      "name": "Centúúm",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cey",
+      "inverted_name": "Chin, Ekai",
+      "name": "Ekai Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cfa",
+      "name": "Dijim-Bwilim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cfd",
+      "name": "Cara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cfg",
+      "name": "Como Karim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cfm",
+      "inverted_name": "Chin, Falam",
+      "name": "Falam Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cga",
+      "name": "Changriwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cgc",
+      "name": "Kagayanen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cgg",
+      "name": "Chiga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cgk",
+      "name": "Chocangacakha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ch",
+      "alpha_3": "cha",
+      "name": "Chamorro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chb",
+      "name": "Chibcha",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "chc",
+      "name": "Catawba",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "chd",
+      "inverted_name": "Chontal, Highland Oaxaca",
+      "name": "Highland Oaxaca Chontal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ce",
+      "alpha_3": "che",
+      "name": "Chechen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chf",
+      "inverted_name": "Chontal, Tabasco",
+      "name": "Tabasco Chontal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chg",
+      "name": "Chagatai",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "chh",
+      "name": "Chinook",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "chj",
+      "inverted_name": "Chinantec, Ojitlán",
+      "name": "Ojitlán Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chk",
+      "name": "Chuukese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chl",
+      "name": "Cahuilla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chm",
+      "name": "Mari (Russia)",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chn",
+      "name": "Chinook jargon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cho",
+      "name": "Choctaw",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chp",
+      "name": "Chipewyan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chq",
+      "inverted_name": "Chinantec, Quiotepec",
+      "name": "Quiotepec Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chr",
+      "name": "Cherokee",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cht",
+      "name": "Cholón",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_2": "cu",
+      "alpha_3": "chu",
+      "inverted_name": "Slavic, Church",
+      "name": "Church Slavic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_2": "cv",
+      "alpha_3": "chv",
+      "name": "Chuvash",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chw",
+      "name": "Chuwabu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chx",
+      "name": "Chantyal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chy",
+      "name": "Cheyenne",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "chz",
+      "inverted_name": "Chinantec, Ozumacín",
+      "name": "Ozumacín Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cia",
+      "name": "Cia-Cia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cib",
+      "inverted_name": "Gbe, Ci",
+      "name": "Ci Gbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cic",
+      "name": "Chickasaw",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cid",
+      "name": "Chimariko",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "cie",
+      "name": "Cineni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cih",
+      "name": "Chinali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cik",
+      "inverted_name": "Kinnauri, Chitkuli",
+      "name": "Chitkuli Kinnauri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cim",
+      "name": "Cimbrian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cin",
+      "name": "Cinta Larga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cip",
+      "name": "Chiapanec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cir",
+      "name": "Tiri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ciw",
+      "name": "Chippewa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ciy",
+      "name": "Chaima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cja",
+      "inverted_name": "Cham, Western",
+      "name": "Western Cham",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cje",
+      "name": "Chru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cjh",
+      "inverted_name": "Chehalis, Upper",
+      "name": "Upper Chehalis",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "cji",
+      "name": "Chamalal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cjk",
+      "name": "Chokwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cjm",
+      "inverted_name": "Cham, Eastern",
+      "name": "Eastern Cham",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cjn",
+      "name": "Chenapian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cjo",
+      "name": "Ashéninka Pajonal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cjp",
+      "name": "Cabécar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cjs",
+      "name": "Shor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cjv",
+      "name": "Chuave",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cjy",
+      "inverted_name": "Chinese, Jinyu",
+      "name": "Jinyu Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ckb",
+      "inverted_name": "Kurdish, Central",
+      "name": "Central Kurdish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ckh",
+      "name": "Chak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ckl",
+      "name": "Cibak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ckm",
+      "name": "Chakavian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ckn",
+      "inverted_name": "Chin, Kaang",
+      "name": "Kaang Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cko",
+      "name": "Anufo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ckq",
+      "name": "Kajakse",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ckr",
+      "name": "Kairak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cks",
+      "name": "Tayo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ckt",
+      "name": "Chukot",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cku",
+      "name": "Koasati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ckv",
+      "name": "Kavalan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ckx",
+      "name": "Caka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cky",
+      "name": "Cakfem-Mushere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ckz",
+      "name": "Cakchiquel-Quiché Mixed Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cla",
+      "name": "Ron",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "clc",
+      "name": "Chilcotin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cld",
+      "inverted_name": "Neo-Aramaic, Chaldean",
+      "name": "Chaldean Neo-Aramaic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cle",
+      "inverted_name": "Chinantec, Lealao",
+      "name": "Lealao Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "clh",
+      "name": "Chilisso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cli",
+      "name": "Chakali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "clj",
+      "inverted_name": "Chin, Laitu",
+      "name": "Laitu Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "clk",
+      "name": "Idu-Mishmi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cll",
+      "name": "Chala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "clm",
+      "name": "Clallam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "clo",
+      "inverted_name": "Chontal, Lowland Oaxaca",
+      "name": "Lowland Oaxaca Chontal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "clt",
+      "inverted_name": "Chin, Lautu",
+      "name": "Lautu Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "clu",
+      "name": "Caluyanun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "clw",
+      "name": "Chulym",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cly",
+      "inverted_name": "Chatino, Eastern Highland",
+      "name": "Eastern Highland Chatino",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cma",
+      "name": "Maa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cme",
+      "name": "Cerma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cmg",
+      "inverted_name": "Mongolian, Classical",
+      "name": "Classical Mongolian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "cmi",
+      "name": "Emberá-Chamí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cml",
+      "name": "Campalagian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cmm",
+      "name": "Michigamea",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "cmn",
+      "inverted_name": "Chinese, Mandarin",
+      "name": "Mandarin Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cmo",
+      "inverted_name": "Mnong, Central",
+      "name": "Central Mnong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cmr",
+      "inverted_name": "Chin, Mro-Khimi",
+      "name": "Mro-Khimi Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cms",
+      "name": "Messapic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "cmt",
+      "name": "Camtho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cna",
+      "name": "Changthang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cnb",
+      "inverted_name": "Chin, Chinbon",
+      "name": "Chinbon Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cnc",
+      "name": "Côông",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cng",
+      "inverted_name": "Qiang, Northern",
+      "name": "Northern Qiang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cnh",
+      "inverted_name": "Chin, Hakha",
+      "name": "Hakha Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cni",
+      "name": "Asháninka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cnk",
+      "inverted_name": "Chin, Khumi",
+      "name": "Khumi Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cnl",
+      "inverted_name": "Chinantec, Lalana",
+      "name": "Lalana Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cno",
+      "name": "Con",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cnp",
+      "inverted_name": "Chinese, Northern Ping",
+      "name": "Northern Ping Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cnq",
+      "name": "Chung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cnr",
+      "name": "Montenegrin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cns",
+      "inverted_name": "Asmat, Central",
+      "name": "Central Asmat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cnt",
+      "inverted_name": "Chinantec, Tepetotutla",
+      "name": "Tepetotutla Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cnu",
+      "name": "Chenoua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cnw",
+      "inverted_name": "Chin, Ngawn",
+      "name": "Ngawn Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cnx",
+      "inverted_name": "Cornish, Middle",
+      "name": "Middle Cornish",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "coa",
+      "inverted_name": "Malay, Cocos Islands",
+      "name": "Cocos Islands Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cob",
+      "name": "Chicomuceltec",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "coc",
+      "name": "Cocopa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cod",
+      "name": "Cocama-Cocamilla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "coe",
+      "name": "Koreguaje",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cof",
+      "name": "Colorado",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cog",
+      "name": "Chong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "coh",
+      "name": "Chonyi-Dzihana-Kauma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "coj",
+      "name": "Cochimi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "cok",
+      "inverted_name": "Cora, Santa Teresa",
+      "name": "Santa Teresa Cora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "col",
+      "name": "Columbia-Wenatchi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "com",
+      "name": "Comanche",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "con",
+      "name": "Cofán",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "coo",
+      "name": "Comox",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cop",
+      "name": "Coptic",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "coq",
+      "name": "Coquille",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_2": "kw",
+      "alpha_3": "cor",
+      "name": "Cornish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "co",
+      "alpha_3": "cos",
+      "name": "Corsican",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cot",
+      "name": "Caquinte",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cou",
+      "name": "Wamey",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cov",
+      "name": "Cao Miao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cow",
+      "name": "Cowlitz",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "cox",
+      "name": "Nanti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "coz",
+      "name": "Chochotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cpa",
+      "inverted_name": "Chinantec, Palantla",
+      "name": "Palantla Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cpb",
+      "inverted_name": "Ashéninka, Ucayali-Yurúa",
+      "name": "Ucayali-Yurúa Ashéninka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cpc",
+      "name": "Ajyíninka Apurucayali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cpg",
+      "inverted_name": "Greek, Cappadocian",
+      "name": "Cappadocian Greek",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "cpi",
+      "inverted_name": "Pidgin English, Chinese",
+      "name": "Chinese Pidgin English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cpn",
+      "name": "Cherepon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cpo",
+      "name": "Kpeego",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cps",
+      "name": "Capiznon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cpu",
+      "inverted_name": "Ashéninka, Pichis",
+      "name": "Pichis Ashéninka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cpx",
+      "inverted_name": "Chinese, Pu-Xian",
+      "name": "Pu-Xian Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cpy",
+      "inverted_name": "Ashéninka, South Ucayali",
+      "name": "South Ucayali Ashéninka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cqd",
+      "inverted_name": "Miao, Chuanqiandian Cluster",
+      "name": "Chuanqiandian Cluster Miao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cra",
+      "name": "Chara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crb",
+      "inverted_name": "Carib, Island",
+      "name": "Island Carib",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "crc",
+      "name": "Lonwolwol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crd",
+      "name": "Coeur d'Alene",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "cr",
+      "alpha_3": "cre",
+      "name": "Cree",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crf",
+      "name": "Caramanta",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "crg",
+      "name": "Michif",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crh",
+      "inverted_name": "Tatar, Crimean",
+      "name": "Crimean Tatar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cri",
+      "name": "Sãotomense",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crj",
+      "inverted_name": "Cree, Southern East",
+      "name": "Southern East Cree",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crk",
+      "inverted_name": "Cree, Plains",
+      "name": "Plains Cree",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crl",
+      "inverted_name": "Cree, Northern East",
+      "name": "Northern East Cree",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crm",
+      "inverted_name": "Cree, Moose",
+      "name": "Moose Cree",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crn",
+      "inverted_name": "Cora, El Nayar",
+      "name": "El Nayar Cora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cro",
+      "name": "Crow",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crq",
+      "inverted_name": "Chorote, Iyo'wujwa",
+      "name": "Iyo'wujwa Chorote",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crr",
+      "inverted_name": "Algonquian, Carolina",
+      "name": "Carolina Algonquian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "crs",
+      "inverted_name": "Creole French, Seselwa",
+      "name": "Seselwa Creole French",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crt",
+      "inverted_name": "Chorote, Iyojwa'ja",
+      "name": "Iyojwa'ja Chorote",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crv",
+      "name": "Chaura",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crw",
+      "name": "Chrau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crx",
+      "name": "Carrier",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cry",
+      "name": "Cori",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "crz",
+      "name": "Cruzeño",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "csa",
+      "inverted_name": "Chinantec, Chiltepec",
+      "name": "Chiltepec Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csb",
+      "name": "Kashubian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csc",
+      "name": "Catalan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csd",
+      "name": "Chiangmai Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cse",
+      "name": "Czech Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csf",
+      "name": "Cuba Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csg",
+      "name": "Chilean Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csh",
+      "inverted_name": "Chin, Asho",
+      "name": "Asho Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csi",
+      "inverted_name": "Miwok, Coast",
+      "name": "Coast Miwok",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "csj",
+      "inverted_name": "Chin, Songlai",
+      "name": "Songlai Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csk",
+      "name": "Jola-Kasa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csl",
+      "name": "Chinese Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csm",
+      "inverted_name": "Miwok, Central Sierra",
+      "name": "Central Sierra Miwok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csn",
+      "name": "Colombian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cso",
+      "inverted_name": "Chinantec, Sochiapam",
+      "name": "Sochiapam Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csp",
+      "inverted_name": "Chinese, Southern Ping",
+      "name": "Southern Ping Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csq",
+      "name": "Croatia Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csr",
+      "name": "Costa Rican Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "css",
+      "inverted_name": "Ohlone, Southern",
+      "name": "Southern Ohlone",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "cst",
+      "inverted_name": "Ohlone, Northern",
+      "name": "Northern Ohlone",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csv",
+      "inverted_name": "Chin, Sumtu",
+      "name": "Sumtu Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csw",
+      "inverted_name": "Cree, Swampy",
+      "name": "Swampy Cree",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csx",
+      "name": "Cambodian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csy",
+      "inverted_name": "Chin, Siyin",
+      "name": "Siyin Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "csz",
+      "name": "Coos",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cta",
+      "inverted_name": "Chatino, Tataltepec",
+      "name": "Tataltepec Chatino",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ctc",
+      "name": "Chetco",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ctd",
+      "inverted_name": "Chin, Tedim",
+      "name": "Tedim Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cte",
+      "inverted_name": "Chinantec, Tepinapa",
+      "name": "Tepinapa Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ctg",
+      "name": "Chittagonian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cth",
+      "inverted_name": "Chin, Thaiphum",
+      "name": "Thaiphum Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ctl",
+      "inverted_name": "Chinantec, Tlacoatzintepec",
+      "name": "Tlacoatzintepec Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ctm",
+      "name": "Chitimacha",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ctn",
+      "name": "Chhintange",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cto",
+      "name": "Emberá-Catío",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ctp",
+      "inverted_name": "Chatino, Western Highland",
+      "name": "Western Highland Chatino",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cts",
+      "inverted_name": "Bikol, Northern Catanduanes",
+      "name": "Northern Catanduanes Bikol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ctt",
+      "inverted_name": "Chetti, Wayanad",
+      "name": "Wayanad Chetti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ctu",
+      "name": "Chol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cty",
+      "name": "Moundadan Chetty",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ctz",
+      "inverted_name": "Chatino, Zacatepec",
+      "name": "Zacatepec Chatino",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cua",
+      "name": "Cua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cub",
+      "name": "Cubeo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cuc",
+      "inverted_name": "Chinantec, Usila",
+      "name": "Usila Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cuh",
+      "name": "Chuka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cui",
+      "name": "Cuiba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cuj",
+      "name": "Mashco Piro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cuk",
+      "inverted_name": "Kuna, San Blas",
+      "name": "San Blas Kuna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cul",
+      "name": "Culina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cuo",
+      "name": "Cumanagoto",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "cup",
+      "name": "Cupeño",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "cuq",
+      "name": "Cun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cur",
+      "name": "Chhulung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cut",
+      "inverted_name": "Cuicatec, Teutila",
+      "name": "Teutila Cuicatec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cuu",
+      "name": "Tai Ya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cuv",
+      "name": "Cuvok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cuw",
+      "name": "Chukwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cux",
+      "inverted_name": "Cuicatec, Tepeuxila",
+      "name": "Tepeuxila Cuicatec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cuy",
+      "name": "Cuitlatec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cvg",
+      "name": "Chug",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cvn",
+      "inverted_name": "Chinantec, Valle Nacional",
+      "name": "Valle Nacional Chinantec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cwa",
+      "name": "Kabwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cwb",
+      "name": "Maindo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cwd",
+      "inverted_name": "Cree, Woods",
+      "name": "Woods Cree",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cwe",
+      "name": "Kwere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cwg",
+      "name": "Chewong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cwt",
+      "name": "Kuwaataay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cya",
+      "inverted_name": "Chatino, Nopala",
+      "name": "Nopala Chatino",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cyb",
+      "name": "Cayubaba",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_2": "cy",
+      "alpha_3": "cym",
+      "bibliographic": "wel",
+      "name": "Welsh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "cyo",
+      "name": "Cuyonon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "czh",
+      "inverted_name": "Chinese, Huizhou",
+      "name": "Huizhou Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "czk",
+      "name": "Knaanic",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "czn",
+      "inverted_name": "Chatino, Zenzontepec",
+      "name": "Zenzontepec Chatino",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "czo",
+      "inverted_name": "Chinese, Min Zhong",
+      "name": "Min Zhong Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "czt",
+      "inverted_name": "Chin, Zotung",
+      "name": "Zotung Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "daa",
+      "name": "Dangaléat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dac",
+      "name": "Dambi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dad",
+      "name": "Marik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dae",
+      "name": "Duupa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dag",
+      "name": "Dagbani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dah",
+      "name": "Gwahatike",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dai",
+      "name": "Day",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "daj",
+      "inverted_name": "Daju, Dar Fur",
+      "name": "Dar Fur Daju",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dak",
+      "name": "Dakota",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dal",
+      "name": "Dahalo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dam",
+      "name": "Damakawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "da",
+      "alpha_3": "dan",
+      "name": "Danish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dao",
+      "inverted_name": "Chin, Daai",
+      "name": "Daai Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "daq",
+      "inverted_name": "Maria, Dandami",
+      "name": "Dandami Maria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dar",
+      "name": "Dargwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "das",
+      "name": "Daho-Doo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dau",
+      "inverted_name": "Daju, Dar Sila",
+      "name": "Dar Sila Daju",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dav",
+      "name": "Taita",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "daw",
+      "name": "Davawenyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dax",
+      "name": "Dayi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "daz",
+      "name": "Dao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dba",
+      "name": "Bangime",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbb",
+      "name": "Deno",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbd",
+      "name": "Dadiya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbe",
+      "name": "Dabe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbf",
+      "name": "Edopi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbg",
+      "inverted_name": "Dogon, Dogul Dom",
+      "name": "Dogul Dom Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbi",
+      "name": "Doka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbj",
+      "name": "Ida'an",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbl",
+      "name": "Dyirbal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbm",
+      "name": "Duguri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbn",
+      "name": "Duriankere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbo",
+      "name": "Dulbu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbp",
+      "name": "Duwai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbq",
+      "name": "Daba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbr",
+      "name": "Dabarre",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbt",
+      "inverted_name": "Dogon, Ben Tey",
+      "name": "Ben Tey Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbu",
+      "inverted_name": "Dogon, Bondum Dom",
+      "name": "Bondum Dom Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbv",
+      "name": "Dungu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dbw",
+      "inverted_name": "Dogon, Bankan Tey",
+      "name": "Bankan Tey Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dby",
+      "name": "Dibiyaso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dcc",
+      "name": "Deccan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dcr",
+      "name": "Negerhollands",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dda",
+      "name": "Dadi Dadi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ddd",
+      "name": "Dongotono",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dde",
+      "name": "Doondo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ddg",
+      "name": "Fataluku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ddi",
+      "inverted_name": "Goodenough, West",
+      "name": "West Goodenough",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ddj",
+      "name": "Jaru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ddn",
+      "name": "Dendi (Benin)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ddo",
+      "name": "Dido",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ddr",
+      "name": "Dhudhuroa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dds",
+      "inverted_name": "Dogon, Donno So",
+      "name": "Donno So Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ddw",
+      "name": "Dawera-Daweloor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dec",
+      "name": "Dagik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ded",
+      "name": "Dedua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dee",
+      "name": "Dewoin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "def",
+      "name": "Dezfuli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "deg",
+      "name": "Degema",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "deh",
+      "name": "Dehwari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dei",
+      "name": "Demisa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dek",
+      "name": "Dek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "del",
+      "name": "Delaware",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dem",
+      "name": "Dem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "den",
+      "name": "Slave (Athapascan)",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dep",
+      "inverted_name": "Delaware, Pidgin",
+      "name": "Pidgin Delaware",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "deq",
+      "name": "Dendi (Central African Republic)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "der",
+      "name": "Deori",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "des",
+      "name": "Desano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "de",
+      "alpha_3": "deu",
+      "bibliographic": "ger",
+      "name": "German",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dev",
+      "name": "Domung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dez",
+      "name": "Dengese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dga",
+      "inverted_name": "Dagaare, Southern",
+      "name": "Southern Dagaare",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dgb",
+      "inverted_name": "Dogon, Bunoge",
+      "name": "Bunoge Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dgc",
+      "inverted_name": "Agta, Casiguran Dumagat",
+      "name": "Casiguran Dumagat Agta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dgd",
+      "name": "Dagaari Dioula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dge",
+      "name": "Degenan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dgg",
+      "name": "Doga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dgh",
+      "name": "Dghwede",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dgi",
+      "inverted_name": "Dagara, Northern",
+      "name": "Northern Dagara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dgk",
+      "name": "Dagba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dgl",
+      "name": "Andaandi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dgn",
+      "name": "Dagoman",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dgo",
+      "name": "Dogri (individual language)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dgr",
+      "name": "Dogrib",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dgs",
+      "name": "Dogoso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dgt",
+      "name": "Ndra'ngith",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dgw",
+      "name": "Daungwurrung",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dgx",
+      "name": "Doghoro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dgz",
+      "name": "Daga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dhd",
+      "name": "Dhundari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dhg",
+      "name": "Dhangu-Djangu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dhi",
+      "name": "Dhimal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dhl",
+      "name": "Dhalandji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dhm",
+      "name": "Zemba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dhn",
+      "name": "Dhanki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dho",
+      "name": "Dhodia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dhr",
+      "name": "Dhargari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dhs",
+      "name": "Dhaiso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dhu",
+      "name": "Dhurga",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dhv",
+      "name": "Dehu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dhw",
+      "name": "Dhanwar (Nepal)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dhx",
+      "name": "Dhungaloo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dia",
+      "name": "Dia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dib",
+      "inverted_name": "Dinka, South Central",
+      "name": "South Central Dinka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dic",
+      "inverted_name": "Dida, Lakota",
+      "name": "Lakota Dida",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "did",
+      "name": "Didinga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dif",
+      "name": "Dieri",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dig",
+      "name": "Digo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dih",
+      "name": "Kumiai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dii",
+      "name": "Dimbong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dij",
+      "name": "Dai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dik",
+      "inverted_name": "Dinka, Southwestern",
+      "name": "Southwestern Dinka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dil",
+      "name": "Dilling",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dim",
+      "name": "Dime",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "din",
+      "name": "Dinka",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dio",
+      "name": "Dibo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dip",
+      "inverted_name": "Dinka, Northeastern",
+      "name": "Northeastern Dinka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "diq",
+      "name": "Dimli (individual language)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dir",
+      "name": "Dirim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dis",
+      "name": "Dimasa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "diu",
+      "name": "Diriku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "dv",
+      "alpha_3": "div",
+      "name": "Dhivehi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "diw",
+      "inverted_name": "Dinka, Northwestern",
+      "name": "Northwestern Dinka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dix",
+      "name": "Dixon Reef",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "diy",
+      "name": "Diuwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "diz",
+      "name": "Ding",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dja",
+      "name": "Djadjawurrung",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "djb",
+      "name": "Djinba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "djc",
+      "inverted_name": "Daju, Dar Daju",
+      "name": "Dar Daju Daju",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "djd",
+      "name": "Djamindjung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dje",
+      "name": "Zarma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "djf",
+      "name": "Djangun",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dji",
+      "name": "Djinang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "djj",
+      "name": "Djeebbana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "djk",
+      "name": "Eastern Maroon Creole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "djm",
+      "inverted_name": "Dogon, Jamsay",
+      "name": "Jamsay Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "djn",
+      "name": "Jawoyn",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "djo",
+      "name": "Jangkang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "djr",
+      "name": "Djambarrpuyngu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dju",
+      "name": "Kapriman",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "djw",
+      "name": "Djawi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dka",
+      "name": "Dakpakha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dkg",
+      "name": "Kadung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dkk",
+      "name": "Dakka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dkr",
+      "name": "Kuijau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dks",
+      "inverted_name": "Dinka, Southeastern",
+      "name": "Southeastern Dinka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dkx",
+      "name": "Mazagway",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dlg",
+      "name": "Dolgan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dlk",
+      "name": "Dahalik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dlm",
+      "name": "Dalmatian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dln",
+      "name": "Darlong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dma",
+      "name": "Duma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dmb",
+      "inverted_name": "Dogon, Mombo",
+      "name": "Mombo Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dmc",
+      "name": "Gavak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dmd",
+      "name": "Madhi Madhi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dme",
+      "name": "Dugwor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dmf",
+      "name": "Medefaidrin",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dmg",
+      "inverted_name": "Kinabatangan, Upper",
+      "name": "Upper Kinabatangan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dmk",
+      "name": "Domaaki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dml",
+      "name": "Dameli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dmm",
+      "name": "Dama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dmo",
+      "name": "Kemedzung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dmr",
+      "inverted_name": "Damar, East",
+      "name": "East Damar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dms",
+      "name": "Dampelas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dmu",
+      "name": "Dubu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dmv",
+      "name": "Dumpas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dmw",
+      "name": "Mudburra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dmx",
+      "name": "Dema",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dmy",
+      "name": "Demta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dna",
+      "inverted_name": "Dani, Upper Grand Valley",
+      "name": "Upper Grand Valley Dani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dnd",
+      "name": "Daonda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dne",
+      "name": "Ndendeule",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dng",
+      "name": "Dungan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dni",
+      "inverted_name": "Dani, Lower Grand Valley",
+      "name": "Lower Grand Valley Dani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dnj",
+      "name": "Dan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dnk",
+      "name": "Dengka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dnn",
+      "name": "Dzùùngoo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dno",
+      "name": "Ndrulo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dnr",
+      "name": "Danaru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dnt",
+      "inverted_name": "Dani, Mid Grand Valley",
+      "name": "Mid Grand Valley Dani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dnu",
+      "name": "Danau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dnv",
+      "name": "Danu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dnw",
+      "inverted_name": "Dani, Western",
+      "name": "Western Dani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dny",
+      "name": "Dení",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "doa",
+      "name": "Dom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dob",
+      "name": "Dobu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "doc",
+      "inverted_name": "Dong, Northern",
+      "name": "Northern Dong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "doe",
+      "name": "Doe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dof",
+      "name": "Domu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "doh",
+      "name": "Dong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "doi",
+      "name": "Dogri (macrolanguage)",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dok",
+      "name": "Dondo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dol",
+      "name": "Doso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "don",
+      "name": "Toura (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "doo",
+      "name": "Dongo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dop",
+      "name": "Lukpa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "doq",
+      "name": "Dominican Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dor",
+      "name": "Dori'o",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dos",
+      "name": "Dogosé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dot",
+      "name": "Dass",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dov",
+      "name": "Dombe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dow",
+      "name": "Doyayo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dox",
+      "name": "Bussa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "doy",
+      "name": "Dompo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "doz",
+      "name": "Dorze",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dpp",
+      "name": "Papar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "drb",
+      "name": "Dair",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "drc",
+      "name": "Minderico",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "drd",
+      "name": "Darmiya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dre",
+      "name": "Dolpo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "drg",
+      "name": "Rungus",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dri",
+      "name": "C'Lela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "drl",
+      "name": "Paakantyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "drn",
+      "inverted_name": "Damar, West",
+      "name": "West Damar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dro",
+      "inverted_name": "Melanau, Daro-Matu",
+      "name": "Daro-Matu Melanau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "drq",
+      "name": "Dura",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "drs",
+      "name": "Gedeo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "drt",
+      "name": "Drents",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dru",
+      "name": "Rukai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dry",
+      "name": "Darai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dsb",
+      "inverted_name": "Sorbian, Lower",
+      "name": "Lower Sorbian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dse",
+      "name": "Dutch Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dsh",
+      "name": "Daasanach",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dsi",
+      "name": "Disa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dsl",
+      "name": "Danish Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dsn",
+      "name": "Dusner",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dso",
+      "name": "Desiya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dsq",
+      "name": "Tadaksahak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dsz",
+      "name": "Mardin Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dta",
+      "name": "Daur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dtb",
+      "inverted_name": "Kadazan, Labuk-Kinabatangan",
+      "name": "Labuk-Kinabatangan Kadazan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dtd",
+      "name": "Ditidaht",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dth",
+      "name": "Adithinngithigh",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dti",
+      "inverted_name": "Dogon, Ana Tinga",
+      "name": "Ana Tinga Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dtk",
+      "inverted_name": "Dogon, Tene Kan",
+      "name": "Tene Kan Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dtm",
+      "inverted_name": "Dogon, Tomo Kan",
+      "name": "Tomo Kan Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dtn",
+      "name": "Daatsʼíin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dto",
+      "inverted_name": "Dogon, Tommo So",
+      "name": "Tommo So Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dtp",
+      "inverted_name": "Dusun, Kadazan",
+      "name": "Kadazan Dusun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dtr",
+      "name": "Lotud",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dts",
+      "inverted_name": "Dogon, Toro So",
+      "name": "Toro So Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dtt",
+      "inverted_name": "Dogon, Toro Tegu",
+      "name": "Toro Tegu Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dtu",
+      "inverted_name": "Dogon, Tebul Ure",
+      "name": "Tebul Ure Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dty",
+      "name": "Dotyali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dua",
+      "name": "Duala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dub",
+      "name": "Dubli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "duc",
+      "name": "Duna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "due",
+      "inverted_name": "Agta, Umiray Dumaget",
+      "name": "Umiray Dumaget Agta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "duf",
+      "name": "Dumbea",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dug",
+      "name": "Duruma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "duh",
+      "name": "Dungra Bhil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dui",
+      "name": "Dumun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "duk",
+      "name": "Uyajitaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dul",
+      "inverted_name": "Agta, Alabat Island",
+      "name": "Alabat Island Agta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dum",
+      "inverted_name": "Dutch, Middle (ca. 1050-1350)",
+      "name": "Middle Dutch (ca. 1050-1350)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "dun",
+      "name": "Dusun Deyah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "duo",
+      "inverted_name": "Agta, Dupaninan",
+      "name": "Dupaninan Agta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dup",
+      "name": "Duano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "duq",
+      "name": "Dusun Malang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dur",
+      "name": "Dii",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dus",
+      "name": "Dumi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "duu",
+      "name": "Drung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "duv",
+      "name": "Duvle",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "duw",
+      "name": "Dusun Witu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dux",
+      "name": "Duungooma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "duy",
+      "inverted_name": "Agta, Dicamay",
+      "name": "Dicamay Agta",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "duz",
+      "name": "Duli-Gey",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dva",
+      "name": "Duau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dwa",
+      "name": "Diri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dwk",
+      "inverted_name": "Kui, Dawik",
+      "name": "Dawik Kui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dwr",
+      "name": "Dawro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dws",
+      "name": "Dutton World Speedwords",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "dwu",
+      "name": "Dhuwal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dww",
+      "name": "Dawawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dwy",
+      "name": "Dhuwaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dwz",
+      "inverted_name": "Rai, Dewas",
+      "name": "Dewas Rai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dya",
+      "name": "Dyan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dyb",
+      "name": "Dyaberdyaber",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dyd",
+      "name": "Dyugun",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dyg",
+      "inverted_name": "Agta, Villa Viciosa",
+      "name": "Villa Viciosa Agta",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dyi",
+      "inverted_name": "Senoufo, Djimini",
+      "name": "Djimini Senoufo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dym",
+      "inverted_name": "Dogon, Yanda Dom",
+      "name": "Yanda Dom Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dyn",
+      "name": "Dyangadi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dyo",
+      "name": "Jola-Fonyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dyu",
+      "name": "Dyula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dyy",
+      "name": "Djabugay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dza",
+      "name": "Tunzu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dze",
+      "name": "Djiwarli",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "dzg",
+      "name": "Dazaga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dzl",
+      "name": "Dzalakha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "dzn",
+      "name": "Dzando",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "dz",
+      "alpha_3": "dzo",
+      "name": "Dzongkha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eaa",
+      "name": "Karenggapa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ebc",
+      "name": "Beginci",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ebg",
+      "name": "Ebughu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ebk",
+      "inverted_name": "Bontok, Eastern",
+      "name": "Eastern Bontok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ebo",
+      "name": "Teke-Ebo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ebr",
+      "name": "Ebrié",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ebu",
+      "name": "Embu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ecr",
+      "name": "Eteocretan",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "ecs",
+      "name": "Ecuadorian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ecy",
+      "name": "Eteocypriot",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "eee",
+      "name": "E",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "efa",
+      "name": "Efai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "efe",
+      "name": "Efe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "efi",
+      "name": "Efik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ega",
+      "name": "Ega",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "egl",
+      "name": "Emilian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "egm",
+      "name": "Benamanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ego",
+      "name": "Eggon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "egy",
+      "name": "Egyptian (Ancient)",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "ehs",
+      "name": "Miyakubo Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ehu",
+      "name": "Ehueun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eip",
+      "name": "Eipomek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eit",
+      "name": "Eitiep",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eiv",
+      "name": "Askopan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eja",
+      "name": "Ejamat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eka",
+      "name": "Ekajuk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eke",
+      "name": "Ekit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ekg",
+      "name": "Ekari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eki",
+      "name": "Eki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ekk",
+      "inverted_name": "Estonian, Standard",
+      "name": "Standard Estonian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ekl",
+      "name": "Kol (Bangladesh)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ekm",
+      "name": "Elip",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eko",
+      "name": "Koti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ekp",
+      "name": "Ekpeye",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ekr",
+      "name": "Yace",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eky",
+      "inverted_name": "Kayah, Eastern",
+      "name": "Eastern Kayah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ele",
+      "name": "Elepi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "elh",
+      "name": "El Hugeirat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eli",
+      "name": "Nding",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "elk",
+      "name": "Elkei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "el",
+      "alpha_3": "ell",
+      "bibliographic": "gre",
+      "inverted_name": "Greek, Modern (1453-)",
+      "name": "Modern Greek (1453-)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "elm",
+      "name": "Eleme",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "elo",
+      "name": "El Molo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "elu",
+      "name": "Elu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "elx",
+      "name": "Elamite",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "ema",
+      "name": "Emai-Iuleha-Ora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "emb",
+      "name": "Embaloh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eme",
+      "name": "Emerillon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "emg",
+      "inverted_name": "Meohang, Eastern",
+      "name": "Eastern Meohang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "emi",
+      "name": "Mussau-Emira",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "emk",
+      "inverted_name": "Maninkakan, Eastern",
+      "name": "Eastern Maninkakan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "emm",
+      "name": "Mamulique",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "emn",
+      "name": "Eman",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "emp",
+      "inverted_name": "Emberá, Northern",
+      "name": "Northern Emberá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "emq",
+      "inverted_name": "Minyag, Eastern",
+      "name": "Eastern Minyag",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ems",
+      "inverted_name": "Yupik, Pacific Gulf",
+      "name": "Pacific Gulf Yupik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "emu",
+      "inverted_name": "Muria, Eastern",
+      "name": "Eastern Muria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "emw",
+      "name": "Emplawas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "emx",
+      "name": "Erromintxela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "emy",
+      "inverted_name": "Mayan, Epigraphic",
+      "name": "Epigraphic Mayan",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "emz",
+      "name": "Mbessa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ena",
+      "name": "Apali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "enb",
+      "name": "Markweeta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "enc",
+      "name": "En",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "end",
+      "name": "Ende",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "enf",
+      "inverted_name": "Enets, Forest",
+      "name": "Forest Enets",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "en",
+      "alpha_3": "eng",
+      "name": "English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "enh",
+      "inverted_name": "Enets, Tundra",
+      "name": "Tundra Enets",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "enl",
+      "name": "Enlhet",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "enm",
+      "inverted_name": "English, Middle (1100-1500)",
+      "name": "Middle English (1100-1500)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "enn",
+      "name": "Engenni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eno",
+      "name": "Enggano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "enq",
+      "name": "Enga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "enr",
+      "name": "Emumu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "enu",
+      "name": "Enu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "env",
+      "name": "Enwan (Edo State)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "enw",
+      "name": "Enwan (Akwa Ibom State)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "enx",
+      "name": "Enxet",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eot",
+      "name": "Beti (Côte d'Ivoire)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "epi",
+      "name": "Epie",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "eo",
+      "alpha_3": "epo",
+      "name": "Esperanto",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "era",
+      "name": "Eravallan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "erg",
+      "name": "Sie",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "erh",
+      "name": "Eruwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eri",
+      "name": "Ogea",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "erk",
+      "inverted_name": "Efate, South",
+      "name": "South Efate",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ero",
+      "name": "Horpa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "err",
+      "name": "Erre",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ers",
+      "name": "Ersu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ert",
+      "name": "Eritai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "erw",
+      "name": "Erokwanas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ese",
+      "name": "Ese Ejja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "esg",
+      "inverted_name": "Gondi, Aheri",
+      "name": "Aheri Gondi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "esh",
+      "name": "Eshtehardi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "esi",
+      "inverted_name": "Inupiatun, North Alaskan",
+      "name": "North Alaskan Inupiatun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "esk",
+      "inverted_name": "Inupiatun, Northwest Alaska",
+      "name": "Northwest Alaska Inupiatun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "esl",
+      "name": "Egypt Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "esm",
+      "name": "Esuma",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "esn",
+      "name": "Salvadoran Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eso",
+      "name": "Estonian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "esq",
+      "name": "Esselen",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ess",
+      "inverted_name": "Yupik, Central Siberian",
+      "name": "Central Siberian Yupik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "et",
+      "alpha_3": "est",
+      "name": "Estonian",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "esu",
+      "inverted_name": "Yupik, Central",
+      "name": "Central Yupik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "esy",
+      "name": "Eskayan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "etb",
+      "name": "Etebi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "etc",
+      "name": "Etchemin",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "eth",
+      "name": "Ethiopian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "etn",
+      "name": "Eton (Vanuatu)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eto",
+      "name": "Eton (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "etr",
+      "name": "Edolo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ets",
+      "name": "Yekhee",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ett",
+      "name": "Etruscan",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "etu",
+      "name": "Ejagham",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "etx",
+      "name": "Eten",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "etz",
+      "name": "Semimi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "eu",
+      "alpha_3": "eus",
+      "bibliographic": "baq",
+      "name": "Basque",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eve",
+      "name": "Even",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "evh",
+      "name": "Uvbie",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "evn",
+      "name": "Evenki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ee",
+      "alpha_3": "ewe",
+      "name": "Ewe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ewo",
+      "name": "Ewondo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ext",
+      "name": "Extremaduran",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eya",
+      "name": "Eyak",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "eyo",
+      "name": "Keiyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eza",
+      "name": "Ezaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "eze",
+      "name": "Uzekwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "faa",
+      "name": "Fasu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fab",
+      "name": "Fa d'Ambu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fad",
+      "name": "Wagi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "faf",
+      "name": "Fagani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fag",
+      "name": "Finongan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fah",
+      "inverted_name": "Fali, Baissa",
+      "name": "Baissa Fali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fai",
+      "name": "Faiwol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "faj",
+      "name": "Faita",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fak",
+      "name": "Fang (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fal",
+      "inverted_name": "Fali, South",
+      "name": "South Fali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fam",
+      "name": "Fam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fan",
+      "name": "Fang (Equatorial Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "fo",
+      "alpha_3": "fao",
+      "name": "Faroese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fap",
+      "name": "Paloor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "far",
+      "name": "Fataleka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "fa",
+      "alpha_3": "fas",
+      "bibliographic": "per",
+      "name": "Persian",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fat",
+      "name": "Fanti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fau",
+      "name": "Fayu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fax",
+      "name": "Fala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fay",
+      "inverted_name": "Fars, Southwestern",
+      "name": "Southwestern Fars",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "faz",
+      "inverted_name": "Fars, Northwestern",
+      "name": "Northwestern Fars",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fbl",
+      "inverted_name": "Bikol, West Albay",
+      "name": "West Albay Bikol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fcs",
+      "name": "Quebec Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fer",
+      "name": "Feroge",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ffi",
+      "name": "Foia Foia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ffm",
+      "inverted_name": "Fulfulde, Maasina",
+      "name": "Maasina Fulfulde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fgr",
+      "name": "Fongoro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fia",
+      "name": "Nobiin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fie",
+      "name": "Fyer",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fif",
+      "name": "Faifi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "fj",
+      "alpha_3": "fij",
+      "name": "Fijian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fil",
+      "name": "Filipino",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "fi",
+      "alpha_3": "fin",
+      "name": "Finnish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fip",
+      "name": "Fipa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fir",
+      "name": "Firan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fit",
+      "inverted_name": "Finnish, Tornedalen",
+      "name": "Tornedalen Finnish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fiw",
+      "name": "Fiwaga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fkk",
+      "name": "Kirya-Konzəl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fkv",
+      "inverted_name": "Finnish, Kven",
+      "name": "Kven Finnish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fla",
+      "name": "Kalispel-Pend d'Oreille",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "flh",
+      "name": "Foau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fli",
+      "name": "Fali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fll",
+      "inverted_name": "Fali, North",
+      "name": "North Fali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fln",
+      "name": "Flinders Island",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "flr",
+      "name": "Fuliiru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fly",
+      "name": "Flaaitaal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fmp",
+      "name": "Fe'fe'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fmu",
+      "inverted_name": "Muria, Far Western",
+      "name": "Far Western Muria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fnb",
+      "name": "Fanbak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fng",
+      "name": "Fanagalo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fni",
+      "name": "Fania",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fod",
+      "name": "Foodo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "foi",
+      "name": "Foi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fom",
+      "name": "Foma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fon",
+      "name": "Fon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "for",
+      "name": "Fore",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fos",
+      "name": "Siraya",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "fpe",
+      "inverted_name": "Creole English, Fernando Po",
+      "name": "Fernando Po Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fqs",
+      "name": "Fas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "fr",
+      "alpha_3": "fra",
+      "bibliographic": "fre",
+      "name": "French",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "frc",
+      "inverted_name": "French, Cajun",
+      "name": "Cajun French",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "frd",
+      "name": "Fordata",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "frk",
+      "name": "Frankish",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "frm",
+      "inverted_name": "French, Middle (ca. 1400-1600)",
+      "name": "Middle French (ca. 1400-1600)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "fro",
+      "inverted_name": "French, Old (842-ca. 1400)",
+      "name": "Old French (842-ca. 1400)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "frp",
+      "name": "Arpitan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "frq",
+      "name": "Forak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "frr",
+      "inverted_name": "Frisian, Northern",
+      "name": "Northern Frisian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "frs",
+      "inverted_name": "Frisian, Eastern",
+      "name": "Eastern Frisian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "frt",
+      "name": "Fortsenal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "fy",
+      "alpha_3": "fry",
+      "inverted_name": "Frisian, Western",
+      "name": "Western Frisian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fse",
+      "name": "Finnish Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fsl",
+      "name": "French Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fss",
+      "name": "Finland-Swedish Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fub",
+      "inverted_name": "Fulfulde, Adamawa",
+      "name": "Adamawa Fulfulde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fuc",
+      "name": "Pulaar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fud",
+      "inverted_name": "Futuna, East",
+      "name": "East Futuna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fue",
+      "inverted_name": "Fulfulde, Borgu",
+      "name": "Borgu Fulfulde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fuf",
+      "name": "Pular",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fuh",
+      "inverted_name": "Fulfulde, Western Niger",
+      "name": "Western Niger Fulfulde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fui",
+      "inverted_name": "Fulfulde, Bagirmi",
+      "name": "Bagirmi Fulfulde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fuj",
+      "name": "Ko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ff",
+      "alpha_3": "ful",
+      "name": "Fulah",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fum",
+      "name": "Fum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fun",
+      "name": "Fulniô",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fuq",
+      "inverted_name": "Fulfulde, Central-Eastern Niger",
+      "name": "Central-Eastern Niger Fulfulde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fur",
+      "name": "Friulian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fut",
+      "name": "Futuna-Aniwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fuu",
+      "name": "Furu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fuv",
+      "inverted_name": "Fulfulde, Nigerian",
+      "name": "Nigerian Fulfulde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fuy",
+      "name": "Fuyug",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fvr",
+      "name": "Fur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fwa",
+      "name": "Fwâi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "fwe",
+      "name": "Fwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gaa",
+      "name": "Ga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gab",
+      "name": "Gabri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gac",
+      "inverted_name": "Great Andamanese, Mixed",
+      "name": "Mixed Great Andamanese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gad",
+      "name": "Gaddang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gae",
+      "name": "Guarequena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gaf",
+      "name": "Gende",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gag",
+      "name": "Gagauz",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gah",
+      "name": "Alekano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gai",
+      "name": "Borei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gaj",
+      "name": "Gadsup",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gak",
+      "name": "Gamkonora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gal",
+      "name": "Galolen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gam",
+      "name": "Kandawo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gan",
+      "inverted_name": "Chinese, Gan",
+      "name": "Gan Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gao",
+      "name": "Gants",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gap",
+      "name": "Gal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gaq",
+      "name": "Gata'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gar",
+      "name": "Galeya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gas",
+      "inverted_name": "Garasia, Adiwasi",
+      "name": "Adiwasi Garasia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gat",
+      "name": "Kenati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gau",
+      "inverted_name": "Gadaba, Mudhili",
+      "name": "Mudhili Gadaba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gaw",
+      "name": "Nobonob",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gax",
+      "inverted_name": "Oromo, Borana-Arsi-Guji",
+      "name": "Borana-Arsi-Guji Oromo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gay",
+      "name": "Gayo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gaz",
+      "inverted_name": "Oromo, West Central",
+      "name": "West Central Oromo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gba",
+      "name": "Gbaya (Central African Republic)",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbb",
+      "name": "Kaytetye",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbd",
+      "name": "Karajarri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbe",
+      "name": "Niksek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbf",
+      "name": "Gaikundi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbg",
+      "name": "Gbanziri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbh",
+      "inverted_name": "Gbe, Defi",
+      "name": "Defi Gbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbi",
+      "name": "Galela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbj",
+      "inverted_name": "Gadaba, Bodo",
+      "name": "Bodo Gadaba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbk",
+      "name": "Gaddi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbl",
+      "name": "Gamit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbm",
+      "name": "Garhwali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbn",
+      "name": "Mo'da",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbo",
+      "inverted_name": "Grebo, Northern",
+      "name": "Northern Grebo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbp",
+      "name": "Gbaya-Bossangoa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbq",
+      "name": "Gbaya-Bozoum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbr",
+      "name": "Gbagyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbs",
+      "inverted_name": "Gbe, Gbesi",
+      "name": "Gbesi Gbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbu",
+      "name": "Gagadu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbv",
+      "name": "Gbanu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbw",
+      "name": "Gabi-Gabi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbx",
+      "inverted_name": "Gbe, Eastern Xwla",
+      "name": "Eastern Xwla Gbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gby",
+      "name": "Gbari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gbz",
+      "inverted_name": "Dari, Zoroastrian",
+      "name": "Zoroastrian Dari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gcc",
+      "name": "Mali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gcd",
+      "name": "Ganggalida",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gce",
+      "name": "Galice",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gcf",
+      "inverted_name": "Creole French, Guadeloupean",
+      "name": "Guadeloupean Creole French",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gcl",
+      "inverted_name": "Creole English, Grenadian",
+      "name": "Grenadian Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gcn",
+      "name": "Gaina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gcr",
+      "inverted_name": "Creole French, Guianese",
+      "name": "Guianese Creole French",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gct",
+      "inverted_name": "German, Colonia Tovar",
+      "name": "Colonia Tovar German",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gda",
+      "inverted_name": "Lohar, Gade",
+      "name": "Gade Lohar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdb",
+      "inverted_name": "Gadaba, Pottangi Ollar",
+      "name": "Pottangi Ollar Gadaba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdc",
+      "name": "Gugu Badhun",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gdd",
+      "name": "Gedaged",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gde",
+      "name": "Gude",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdf",
+      "name": "Guduf-Gava",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdg",
+      "name": "Ga'dang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdh",
+      "name": "Gadjerawang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdi",
+      "name": "Gundi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdj",
+      "name": "Gurdjar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdk",
+      "name": "Gadang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdl",
+      "name": "Dirasha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdm",
+      "name": "Laal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdn",
+      "name": "Umanakaina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdo",
+      "name": "Ghodoberi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdq",
+      "name": "Mehri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdr",
+      "name": "Wipi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gds",
+      "name": "Ghandruk Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdt",
+      "name": "Kungardutyi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gdu",
+      "name": "Gudu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gdx",
+      "name": "Godwari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gea",
+      "name": "Geruma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "geb",
+      "name": "Kire",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gec",
+      "inverted_name": "Grebo, Gboloo",
+      "name": "Gboloo Grebo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ged",
+      "name": "Gade",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gef",
+      "name": "Gerai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "geg",
+      "name": "Gengle",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "geh",
+      "inverted_name": "German, Hutterite",
+      "name": "Hutterite German",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gei",
+      "name": "Gebe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gej",
+      "name": "Gen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gek",
+      "name": "Ywom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gel",
+      "name": "ut-Ma'in",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "geq",
+      "name": "Geme",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ges",
+      "name": "Geser-Gorom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gev",
+      "name": "Eviya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gew",
+      "name": "Gera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gex",
+      "name": "Garre",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gey",
+      "name": "Enya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gez",
+      "name": "Geez",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "gfk",
+      "name": "Patpatar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gft",
+      "name": "Gafat",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gga",
+      "name": "Gao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ggb",
+      "name": "Gbii",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ggd",
+      "name": "Gugadj",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gge",
+      "name": "Gurr-goni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ggg",
+      "name": "Gurgula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ggk",
+      "name": "Kungarakany",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ggl",
+      "name": "Ganglau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ggt",
+      "name": "Gitua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ggu",
+      "name": "Gagu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ggw",
+      "name": "Gogodala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gha",
+      "name": "Ghadamès",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ghc",
+      "inverted_name": "Gaelic, Hiberno-Scottish",
+      "name": "Hiberno-Scottish Gaelic",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "ghe",
+      "inverted_name": "Ghale, Southern",
+      "name": "Southern Ghale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ghh",
+      "inverted_name": "Ghale, Northern",
+      "name": "Northern Ghale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ghk",
+      "inverted_name": "Karen, Geko",
+      "name": "Geko Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ghl",
+      "name": "Ghulfan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ghn",
+      "name": "Ghanongga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gho",
+      "name": "Ghomara",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ghr",
+      "name": "Ghera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ghs",
+      "name": "Guhu-Samane",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ght",
+      "name": "Kuke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gia",
+      "name": "Kija",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gib",
+      "name": "Gibanawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gic",
+      "name": "Gail",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gid",
+      "name": "Gidar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gie",
+      "name": "Gaɓogbo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gig",
+      "name": "Goaria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gih",
+      "name": "Githabul",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gii",
+      "name": "Girirra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gil",
+      "name": "Gilbertese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gim",
+      "name": "Gimi (Eastern Highlands)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gin",
+      "name": "Hinukh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gip",
+      "name": "Gimi (West New Britain)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "giq",
+      "inverted_name": "Gelao, Green",
+      "name": "Green Gelao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gir",
+      "inverted_name": "Gelao, Red",
+      "name": "Red Gelao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gis",
+      "inverted_name": "Giziga, North",
+      "name": "North Giziga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "git",
+      "name": "Gitxsan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "giu",
+      "name": "Mulao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "giw",
+      "inverted_name": "Gelao, White",
+      "name": "White Gelao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gix",
+      "name": "Gilima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "giy",
+      "name": "Giyug",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "giz",
+      "inverted_name": "Giziga, South",
+      "name": "South Giziga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gjk",
+      "inverted_name": "Koli, Kachi",
+      "name": "Kachi Koli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gjm",
+      "name": "Gunditjmara",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gjn",
+      "name": "Gonja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gjr",
+      "name": "Gurindji Kriol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gju",
+      "name": "Gujari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gka",
+      "name": "Guya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gkd",
+      "name": "Magɨ (Madang Province)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gke",
+      "name": "Ndai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gkn",
+      "name": "Gokana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gko",
+      "name": "Kok-Nar",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gkp",
+      "inverted_name": "Kpelle, Guinea",
+      "name": "Guinea Kpelle",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gku",
+      "name": "ǂUngkue",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_2": "gd",
+      "alpha_3": "gla",
+      "inverted_name": "Gaelic, Scottish",
+      "name": "Scottish Gaelic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "glb",
+      "name": "Belning",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "glc",
+      "name": "Bon Gula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gld",
+      "name": "Nanai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ga",
+      "alpha_3": "gle",
+      "name": "Irish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "gl",
+      "alpha_3": "glg",
+      "name": "Galician",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "glh",
+      "inverted_name": "Pashai, Northwest",
+      "name": "Northwest Pashai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "glj",
+      "name": "Gula Iro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "glk",
+      "name": "Gilaki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gll",
+      "name": "Garlali",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "glo",
+      "name": "Galambu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "glr",
+      "name": "Glaro-Twabo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "glu",
+      "name": "Gula (Chad)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "gv",
+      "alpha_3": "glv",
+      "name": "Manx",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "glw",
+      "name": "Glavda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gly",
+      "name": "Gule",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gma",
+      "name": "Gambera",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gmb",
+      "name": "Gula'alaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gmd",
+      "name": "Mághdì",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gmg",
+      "name": "Magɨyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gmh",
+      "inverted_name": "German, Middle High (ca. 1050-1500)",
+      "name": "Middle High German (ca. 1050-1500)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "gml",
+      "inverted_name": "German, Middle Low",
+      "name": "Middle Low German",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "gmm",
+      "name": "Gbaya-Mbodomo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gmn",
+      "name": "Gimnime",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gmr",
+      "name": "Mirning",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gmu",
+      "name": "Gumalu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gmv",
+      "name": "Gamo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gmx",
+      "name": "Magoma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gmy",
+      "inverted_name": "Greek, Mycenaean",
+      "name": "Mycenaean Greek",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "gmz",
+      "name": "Mgbolizhia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gna",
+      "name": "Kaansa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gnb",
+      "name": "Gangte",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gnc",
+      "name": "Guanche",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gnd",
+      "name": "Zulgo-Gemzek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gne",
+      "name": "Ganang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gng",
+      "name": "Ngangam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gnh",
+      "name": "Lere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gni",
+      "name": "Gooniyandi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gnj",
+      "name": "Ngen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gnk",
+      "name": "ǁGana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gnl",
+      "name": "Gangulu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gnm",
+      "name": "Ginuman",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gnn",
+      "name": "Gumatj",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gno",
+      "inverted_name": "Gondi, Northern",
+      "name": "Northern Gondi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gnq",
+      "name": "Gana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gnr",
+      "name": "Gureng Gureng",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gnt",
+      "name": "Guntai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gnu",
+      "name": "Gnau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gnw",
+      "inverted_name": "Guaraní, Western Bolivian",
+      "name": "Western Bolivian Guaraní",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gnz",
+      "name": "Ganzi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "goa",
+      "name": "Guro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gob",
+      "name": "Playero",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "goc",
+      "name": "Gorakor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "god",
+      "name": "Godié",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "goe",
+      "name": "Gongduk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gof",
+      "name": "Gofa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gog",
+      "name": "Gogo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "goh",
+      "inverted_name": "German, Old High (ca. 750-1050)",
+      "name": "Old High German (ca. 750-1050)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "goi",
+      "name": "Gobasi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "goj",
+      "name": "Gowlan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gok",
+      "name": "Gowli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gol",
+      "name": "Gola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gom",
+      "inverted_name": "Konkani, Goan",
+      "name": "Goan Konkani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gon",
+      "name": "Gondi",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "goo",
+      "name": "Gone Dau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gop",
+      "name": "Yeretuar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "goq",
+      "name": "Gorap",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gor",
+      "name": "Gorontalo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gos",
+      "name": "Gronings",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "got",
+      "name": "Gothic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "gou",
+      "name": "Gavar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gov",
+      "name": "Goo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gow",
+      "name": "Gorowa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gox",
+      "name": "Gobu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "goy",
+      "name": "Goundo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "goz",
+      "name": "Gozarkhani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gpa",
+      "name": "Gupa-Abawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gpe",
+      "inverted_name": "Pidgin English, Ghanaian",
+      "name": "Ghanaian Pidgin English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gpn",
+      "name": "Taiap",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gqa",
+      "name": "Ga'anda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gqi",
+      "name": "Guiqiong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gqn",
+      "name": "Guana (Brazil)",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gqr",
+      "name": "Gor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gqu",
+      "name": "Qau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gra",
+      "inverted_name": "Garasia, Rajput",
+      "name": "Rajput Garasia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grb",
+      "name": "Grebo",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grc",
+      "inverted_name": "Greek, Ancient (to 1453)",
+      "name": "Ancient Greek (to 1453)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "grd",
+      "name": "Guruntum-Mbaaru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grg",
+      "name": "Madi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grh",
+      "name": "Gbiri-Niragu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gri",
+      "name": "Ghari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grj",
+      "inverted_name": "Grebo, Southern",
+      "name": "Southern Grebo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grm",
+      "name": "Kota Marudu Talantang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "gn",
+      "alpha_3": "grn",
+      "name": "Guarani",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gro",
+      "name": "Groma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grq",
+      "name": "Gorovu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grr",
+      "name": "Taznatit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grs",
+      "name": "Gresi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grt",
+      "name": "Garo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gru",
+      "name": "Kistane",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grv",
+      "inverted_name": "Grebo, Central",
+      "name": "Central Grebo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grw",
+      "name": "Gweda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grx",
+      "name": "Guriaso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gry",
+      "inverted_name": "Grebo, Barclayville",
+      "name": "Barclayville Grebo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "grz",
+      "name": "Guramalum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gse",
+      "name": "Ghanaian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gsg",
+      "name": "German Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gsl",
+      "name": "Gusilay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gsm",
+      "name": "Guatemalan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gsn",
+      "name": "Nema",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gso",
+      "inverted_name": "Gbaya, Southwest",
+      "name": "Southwest Gbaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gsp",
+      "name": "Wasembo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gss",
+      "name": "Greek Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gsw",
+      "inverted_name": "German, Swiss",
+      "name": "Swiss German",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gta",
+      "name": "Guató",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gtu",
+      "name": "Aghu-Tharnggala",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gua",
+      "name": "Shiki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gub",
+      "name": "Guajajára",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "guc",
+      "name": "Wayuu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gud",
+      "inverted_name": "Dida, Yocoboué",
+      "name": "Yocoboué Dida",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gue",
+      "name": "Gurindji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "guf",
+      "name": "Gupapuyngu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gug",
+      "inverted_name": "Guaraní, Paraguayan",
+      "name": "Paraguayan Guaraní",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "guh",
+      "name": "Guahibo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gui",
+      "inverted_name": "Guaraní, Eastern Bolivian",
+      "name": "Eastern Bolivian Guaraní",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "gu",
+      "alpha_3": "guj",
+      "name": "Gujarati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "guk",
+      "name": "Gumuz",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gul",
+      "inverted_name": "Creole English, Sea Island",
+      "name": "Sea Island Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gum",
+      "name": "Guambiano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gun",
+      "inverted_name": "Guaraní, Mbyá",
+      "name": "Mbyá Guaraní",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "guo",
+      "name": "Guayabero",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gup",
+      "name": "Gunwinggu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "guq",
+      "name": "Aché",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gur",
+      "name": "Farefare",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gus",
+      "name": "Guinean Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gut",
+      "name": "Maléku Jaíka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "guu",
+      "name": "Yanomamö",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "guw",
+      "name": "Gun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gux",
+      "name": "Gourmanchéma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "guz",
+      "name": "Gusii",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gva",
+      "name": "Guana (Paraguay)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gvc",
+      "name": "Guanano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gve",
+      "name": "Duwet",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gvf",
+      "name": "Golin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gvj",
+      "name": "Guajá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gvl",
+      "name": "Gulay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gvm",
+      "name": "Gurmana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gvn",
+      "name": "Kuku-Yalanji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gvo",
+      "name": "Gavião Do Jiparaná",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gvp",
+      "inverted_name": "Gavião, Pará",
+      "name": "Pará Gavião",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gvr",
+      "name": "Gurung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gvs",
+      "name": "Gumawana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gvy",
+      "name": "Guyani",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gwa",
+      "name": "Mbato",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gwb",
+      "name": "Gwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gwc",
+      "name": "Gawri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gwd",
+      "name": "Gawwada",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gwe",
+      "name": "Gweno",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gwf",
+      "name": "Gowro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gwg",
+      "name": "Moo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gwi",
+      "name": "Gwichʼin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gwj",
+      "name": "ǀGwi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gwm",
+      "name": "Awngthim",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gwn",
+      "name": "Gwandara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gwr",
+      "name": "Gwere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gwt",
+      "name": "Gawar-Bati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gwu",
+      "name": "Guwamu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gww",
+      "name": "Kwini",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gwx",
+      "name": "Gua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gxx",
+      "name": "Wè Southern",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gya",
+      "inverted_name": "Gbaya, Northwest",
+      "name": "Northwest Gbaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gyb",
+      "name": "Garus",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gyd",
+      "name": "Kayardild",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gye",
+      "name": "Gyem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gyf",
+      "name": "Gungabula",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gyg",
+      "name": "Gbayi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gyi",
+      "name": "Gyele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gyl",
+      "name": "Gayil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gym",
+      "name": "Ngäbere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gyn",
+      "inverted_name": "Creole English, Guyanese",
+      "name": "Guyanese Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gyo",
+      "name": "Gyalsumdo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gyr",
+      "name": "Guarayu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gyy",
+      "name": "Gunya",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "gyz",
+      "name": "Geji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gza",
+      "name": "Ganza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gzi",
+      "name": "Gazi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "gzn",
+      "name": "Gane",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "haa",
+      "name": "Han",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hab",
+      "name": "Hanoi Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hac",
+      "name": "Gurani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "had",
+      "name": "Hatam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hae",
+      "inverted_name": "Oromo, Eastern",
+      "name": "Eastern Oromo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "haf",
+      "name": "Haiphong Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hag",
+      "name": "Hanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hah",
+      "name": "Hahon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hai",
+      "name": "Haida",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "haj",
+      "name": "Hajong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hak",
+      "inverted_name": "Chinese, Hakka",
+      "name": "Hakka Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hal",
+      "name": "Halang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ham",
+      "name": "Hewa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "han",
+      "name": "Hangaza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hao",
+      "name": "Hakö",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hap",
+      "name": "Hupla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "haq",
+      "name": "Ha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "har",
+      "name": "Harari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "has",
+      "name": "Haisla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ht",
+      "alpha_3": "hat",
+      "name": "Haitian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ha",
+      "alpha_3": "hau",
+      "name": "Hausa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hav",
+      "name": "Havu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "haw",
+      "name": "Hawaiian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hax",
+      "inverted_name": "Haida, Southern",
+      "name": "Southern Haida",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hay",
+      "name": "Haya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "haz",
+      "name": "Hazaragi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hba",
+      "name": "Hamba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hbb",
+      "name": "Huba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hbn",
+      "name": "Heiban",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hbo",
+      "inverted_name": "Hebrew, Ancient",
+      "name": "Ancient Hebrew",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_2": "sh",
+      "alpha_3": "hbs",
+      "name": "Serbo-Croatian",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hbu",
+      "name": "Habu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hca",
+      "inverted_name": "Creole Hindi, Andaman",
+      "name": "Andaman Creole Hindi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hch",
+      "name": "Huichol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hdn",
+      "inverted_name": "Haida, Northern",
+      "name": "Northern Haida",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hds",
+      "name": "Honduras Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hdy",
+      "name": "Hadiyya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hea",
+      "inverted_name": "Miao, Northern Qiandong",
+      "name": "Northern Qiandong Miao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "he",
+      "alpha_3": "heb",
+      "name": "Hebrew",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hed",
+      "name": "Herdé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "heg",
+      "name": "Helong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "heh",
+      "name": "Hehe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hei",
+      "name": "Heiltsuk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hem",
+      "name": "Hemba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "hz",
+      "alpha_3": "her",
+      "name": "Herero",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hgm",
+      "name": "Haiǁom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hgw",
+      "name": "Haigwai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hhi",
+      "name": "Hoia Hoia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hhr",
+      "name": "Kerak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hhy",
+      "name": "Hoyahoya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hia",
+      "name": "Lamang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hib",
+      "name": "Hibito",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "hid",
+      "name": "Hidatsa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hif",
+      "inverted_name": "Hindi, Fiji",
+      "name": "Fiji Hindi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hig",
+      "name": "Kamwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hih",
+      "name": "Pamosu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hii",
+      "name": "Hinduri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hij",
+      "name": "Hijuk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hik",
+      "name": "Seit-Kaitetu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hil",
+      "name": "Hiligaynon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "hi",
+      "alpha_3": "hin",
+      "name": "Hindi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hio",
+      "name": "Tsoa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hir",
+      "name": "Himarimã",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hit",
+      "name": "Hittite",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "hiw",
+      "name": "Hiw",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hix",
+      "name": "Hixkaryána",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hji",
+      "name": "Haji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hka",
+      "name": "Kahe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hke",
+      "name": "Hunde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hkh",
+      "name": "Khah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hkk",
+      "name": "Hunjara-Kaina Ke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hkn",
+      "name": "Mel-Khaonh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hks",
+      "name": "Hong Kong Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hla",
+      "name": "Halia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hlb",
+      "name": "Halbi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hld",
+      "name": "Halang Doan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hle",
+      "name": "Hlersu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hlt",
+      "inverted_name": "Chin, Matu",
+      "name": "Matu Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hlu",
+      "inverted_name": "Luwian, Hieroglyphic",
+      "name": "Hieroglyphic Luwian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "hma",
+      "inverted_name": "Hmong, Southern Mashan",
+      "name": "Southern Mashan Hmong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmb",
+      "inverted_name": "Songhay, Humburi Senni",
+      "name": "Humburi Senni Songhay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmc",
+      "inverted_name": "Hmong, Central Huishui",
+      "name": "Central Huishui Hmong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmd",
+      "inverted_name": "Miao, Large Flowery",
+      "name": "Large Flowery Miao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hme",
+      "inverted_name": "Hmong, Eastern Huishui",
+      "name": "Eastern Huishui Hmong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmf",
+      "name": "Hmong Don",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmg",
+      "inverted_name": "Hmong, Southwestern Guiyang",
+      "name": "Southwestern Guiyang Hmong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmh",
+      "inverted_name": "Hmong, Southwestern Huishui",
+      "name": "Southwestern Huishui Hmong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmi",
+      "inverted_name": "Hmong, Northern Huishui",
+      "name": "Northern Huishui Hmong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmj",
+      "name": "Ge",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmk",
+      "name": "Maek",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "hml",
+      "inverted_name": "Hmong, Luopohe",
+      "name": "Luopohe Hmong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmm",
+      "inverted_name": "Hmong, Central Mashan",
+      "name": "Central Mashan Hmong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmn",
+      "name": "Hmong",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ho",
+      "alpha_3": "hmo",
+      "name": "Hiri Motu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmp",
+      "inverted_name": "Hmong, Northern Mashan",
+      "name": "Northern Mashan Hmong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmq",
+      "inverted_name": "Miao, Eastern Qiandong",
+      "name": "Eastern Qiandong Miao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmr",
+      "name": "Hmar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hms",
+      "inverted_name": "Miao, Southern Qiandong",
+      "name": "Southern Qiandong Miao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmt",
+      "name": "Hamtai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmu",
+      "name": "Hamap",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmv",
+      "name": "Hmong Dô",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmw",
+      "inverted_name": "Hmong, Western Mashan",
+      "name": "Western Mashan Hmong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmy",
+      "inverted_name": "Hmong, Southern Guiyang",
+      "name": "Southern Guiyang Hmong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hmz",
+      "name": "Hmong Shua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hna",
+      "name": "Mina (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hnd",
+      "inverted_name": "Hindko, Southern",
+      "name": "Southern Hindko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hne",
+      "name": "Chhattisgarhi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hng",
+      "name": "Hungu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hnh",
+      "name": "ǁAni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hni",
+      "name": "Hani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hnj",
+      "name": "Hmong Njua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hnn",
+      "name": "Hanunoo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hno",
+      "inverted_name": "Hindko, Northern",
+      "name": "Northern Hindko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hns",
+      "inverted_name": "Hindustani, Caribbean",
+      "name": "Caribbean Hindustani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hnu",
+      "name": "Hung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hoa",
+      "name": "Hoava",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hob",
+      "name": "Mari (Madang Province)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hoc",
+      "name": "Ho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hod",
+      "name": "Holma",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "hoe",
+      "name": "Horom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hoh",
+      "name": "Hobyót",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hoi",
+      "name": "Holikachuk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hoj",
+      "name": "Hadothi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hol",
+      "name": "Holu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hom",
+      "name": "Homa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "hoo",
+      "name": "Holoholo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hop",
+      "name": "Hopi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hor",
+      "name": "Horo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "hos",
+      "name": "Ho Chi Minh City Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hot",
+      "name": "Hote",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hov",
+      "name": "Hovongan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "how",
+      "name": "Honi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hoy",
+      "name": "Holiya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hoz",
+      "name": "Hozo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hpo",
+      "name": "Hpon",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "hps",
+      "name": "Hawai'i Sign Language (HSL)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hra",
+      "name": "Hrangkhol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hrc",
+      "name": "Niwer Mil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hre",
+      "name": "Hre",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hrk",
+      "name": "Haruku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hrm",
+      "inverted_name": "Miao, Horned",
+      "name": "Horned Miao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hro",
+      "name": "Haroi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hrp",
+      "name": "Nhirrpi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "hrt",
+      "name": "Hértevin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hru",
+      "name": "Hruso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "hr",
+      "alpha_3": "hrv",
+      "name": "Croatian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hrw",
+      "name": "Warwar Feni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hrx",
+      "name": "Hunsrik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hrz",
+      "name": "Harzani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hsb",
+      "inverted_name": "Sorbian, Upper",
+      "name": "Upper Sorbian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hsh",
+      "name": "Hungarian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hsl",
+      "name": "Hausa Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hsn",
+      "inverted_name": "Chinese, Xiang",
+      "name": "Xiang Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hss",
+      "name": "Harsusi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hti",
+      "name": "Hoti",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "hto",
+      "inverted_name": "Huitoto, Minica",
+      "name": "Minica Huitoto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hts",
+      "name": "Hadza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "htu",
+      "name": "Hitu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "htx",
+      "inverted_name": "Hittite, Middle",
+      "name": "Middle Hittite",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "hub",
+      "name": "Huambisa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "huc",
+      "name": "ǂHua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hud",
+      "name": "Huaulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hue",
+      "inverted_name": "Huave, San Francisco Del Mar",
+      "name": "San Francisco Del Mar Huave",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "huf",
+      "name": "Humene",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hug",
+      "name": "Huachipaeri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "huh",
+      "name": "Huilliche",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hui",
+      "name": "Huli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "huj",
+      "inverted_name": "Hmong, Northern Guiyang",
+      "name": "Northern Guiyang Hmong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "huk",
+      "name": "Hulung",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "hul",
+      "name": "Hula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hum",
+      "name": "Hungana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "hu",
+      "alpha_3": "hun",
+      "name": "Hungarian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "huo",
+      "name": "Hu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hup",
+      "name": "Hupa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "huq",
+      "name": "Tsat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hur",
+      "name": "Halkomelem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hus",
+      "name": "Huastec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hut",
+      "name": "Humla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "huu",
+      "inverted_name": "Huitoto, Murui",
+      "name": "Murui Huitoto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "huv",
+      "inverted_name": "Huave, San Mateo Del Mar",
+      "name": "San Mateo Del Mar Huave",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "huw",
+      "name": "Hukumina",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "hux",
+      "inverted_name": "Huitoto, Nüpode",
+      "name": "Nüpode Huitoto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "huy",
+      "name": "Hulaulá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "huz",
+      "name": "Hunzib",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hvc",
+      "name": "Haitian Vodoun Culture Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hve",
+      "inverted_name": "Huave, San Dionisio Del Mar",
+      "name": "San Dionisio Del Mar Huave",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hvk",
+      "name": "Haveke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hvn",
+      "name": "Sabu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hvv",
+      "inverted_name": "Huave, Santa María Del Mar",
+      "name": "Santa María Del Mar Huave",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hwa",
+      "name": "Wané",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hwc",
+      "inverted_name": "Creole English, Hawai'i",
+      "name": "Hawai'i Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hwo",
+      "name": "Hwana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hya",
+      "name": "Hya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "hy",
+      "alpha_3": "hye",
+      "bibliographic": "arm",
+      "name": "Armenian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "hyw",
+      "inverted_name": "Armenian, Western",
+      "name": "Western Armenian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iai",
+      "name": "Iaai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ian",
+      "name": "Iatmul",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iar",
+      "name": "Purari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iba",
+      "name": "Iban",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ibb",
+      "name": "Ibibio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ibd",
+      "name": "Iwaidja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ibe",
+      "name": "Akpes",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ibg",
+      "name": "Ibanag",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ibh",
+      "name": "Bih",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ibl",
+      "name": "Ibaloi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ibm",
+      "name": "Agoi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ibn",
+      "name": "Ibino",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ig",
+      "alpha_3": "ibo",
+      "name": "Igbo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ibr",
+      "name": "Ibuoro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ibu",
+      "name": "Ibu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iby",
+      "name": "Ibani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ica",
+      "name": "Ede Ica",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ich",
+      "name": "Etkywan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "icl",
+      "name": "Icelandic Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "icr",
+      "inverted_name": "Creole English, Islander",
+      "name": "Islander Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ida",
+      "name": "Idakho-Isukha-Tiriki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "idb",
+      "name": "Indo-Portuguese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "idc",
+      "name": "Idon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "idd",
+      "name": "Ede Idaca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ide",
+      "name": "Idere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "idi",
+      "name": "Idi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "io",
+      "alpha_3": "ido",
+      "name": "Ido",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "idr",
+      "name": "Indri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ids",
+      "name": "Idesa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "idt",
+      "name": "Idaté",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "idu",
+      "name": "Idoma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ifa",
+      "inverted_name": "Ifugao, Amganad",
+      "name": "Amganad Ifugao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ifb",
+      "inverted_name": "Ifugao, Batad",
+      "name": "Batad Ifugao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ife",
+      "name": "Ifè",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iff",
+      "name": "Ifo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ifk",
+      "inverted_name": "Ifugao, Tuwali",
+      "name": "Tuwali Ifugao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ifm",
+      "name": "Teke-Fuumu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ifu",
+      "inverted_name": "Ifugao, Mayoyao",
+      "name": "Mayoyao Ifugao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ify",
+      "inverted_name": "Kallahan, Keley-I",
+      "name": "Keley-I Kallahan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "igb",
+      "name": "Ebira",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ige",
+      "name": "Igede",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "igg",
+      "name": "Igana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "igl",
+      "name": "Igala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "igm",
+      "name": "Kanggape",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ign",
+      "name": "Ignaciano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "igo",
+      "name": "Isebe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "igs",
+      "name": "Interglossa",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "igw",
+      "name": "Igwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ihb",
+      "name": "Iha Based Pidgin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ihi",
+      "name": "Ihievbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ihp",
+      "name": "Iha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ihw",
+      "name": "Bidhawal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_2": "ii",
+      "alpha_3": "iii",
+      "inverted_name": "Yi, Sichuan",
+      "name": "Sichuan Yi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iin",
+      "name": "Thiin",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ijc",
+      "name": "Izon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ije",
+      "name": "Biseni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ijj",
+      "name": "Ede Ije",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ijn",
+      "name": "Kalabari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ijs",
+      "inverted_name": "Ijo, Southeast",
+      "name": "Southeast Ijo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ike",
+      "inverted_name": "Inuktitut, Eastern Canadian",
+      "name": "Eastern Canadian Inuktitut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iki",
+      "name": "Iko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ikk",
+      "name": "Ika",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ikl",
+      "name": "Ikulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iko",
+      "name": "Olulumo-Ikom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ikp",
+      "name": "Ikpeshi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ikr",
+      "name": "Ikaranggal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "iks",
+      "name": "Inuit Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ikt",
+      "name": "Inuinnaqtun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "iu",
+      "alpha_3": "iku",
+      "name": "Inuktitut",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ikv",
+      "name": "Iku-Gora-Ankwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ikw",
+      "name": "Ikwere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ikx",
+      "name": "Ik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ikz",
+      "name": "Ikizu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ila",
+      "name": "Ile Ape",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ilb",
+      "name": "Ila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ie",
+      "alpha_3": "ile",
+      "name": "Interlingue",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "ilg",
+      "name": "Garig-Ilgar",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ili",
+      "name": "Ili Turki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ilk",
+      "name": "Ilongot",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ilm",
+      "name": "Iranun (Malaysia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ilo",
+      "name": "Iloko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ilp",
+      "name": "Iranun (Philippines)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ils",
+      "name": "International Sign",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ilu",
+      "name": "Ili'uun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ilv",
+      "name": "Ilue",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ima",
+      "inverted_name": "Malasar, Mala",
+      "name": "Mala Malasar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "imi",
+      "name": "Anamgura",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iml",
+      "name": "Miluk",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "imn",
+      "name": "Imonda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "imo",
+      "name": "Imbongu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "imr",
+      "name": "Imroing",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ims",
+      "name": "Marsian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "imt",
+      "name": "Imotong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "imy",
+      "name": "Milyan",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_2": "ia",
+      "alpha_3": "ina",
+      "name": "Interlingua (International Auxiliary Language Association)",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "inb",
+      "name": "Inga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "id",
+      "alpha_3": "ind",
+      "name": "Indonesian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ing",
+      "name": "Degexit'an",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "inh",
+      "name": "Ingush",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "inj",
+      "inverted_name": "Inga, Jungle",
+      "name": "Jungle Inga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "inl",
+      "name": "Indonesian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "inm",
+      "name": "Minaean",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "inn",
+      "name": "Isinai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ino",
+      "name": "Inoke-Yate",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "inp",
+      "name": "Iñapari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ins",
+      "name": "Indian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "int",
+      "name": "Intha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "inz",
+      "name": "Ineseño",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ior",
+      "name": "Inor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iou",
+      "name": "Tuma-Irumu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iow",
+      "name": "Iowa-Oto",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ipi",
+      "name": "Ipili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ik",
+      "alpha_3": "ipk",
+      "name": "Inupiaq",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ipo",
+      "name": "Ipiko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iqu",
+      "name": "Iquito",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iqw",
+      "name": "Ikwo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ire",
+      "name": "Iresim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "irh",
+      "name": "Irarutu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iri",
+      "name": "Rigwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "irk",
+      "name": "Iraqw",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "irn",
+      "name": "Irántxe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "irr",
+      "name": "Ir",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iru",
+      "name": "Irula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "irx",
+      "name": "Kamberau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iry",
+      "name": "Iraya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "isa",
+      "name": "Isabi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "isc",
+      "name": "Isconahua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "isd",
+      "name": "Isnag",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ise",
+      "name": "Italian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "isg",
+      "name": "Irish Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ish",
+      "name": "Esan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "isi",
+      "name": "Nkem-Nkum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "isk",
+      "name": "Ishkashimi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "is",
+      "alpha_3": "isl",
+      "bibliographic": "ice",
+      "name": "Icelandic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ism",
+      "name": "Masimasi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "isn",
+      "name": "Isanzu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iso",
+      "name": "Isoko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "isr",
+      "name": "Israeli Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ist",
+      "name": "Istriot",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "isu",
+      "name": "Isu (Menchum Division)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "it",
+      "alpha_3": "ita",
+      "name": "Italian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "itb",
+      "inverted_name": "Itneg, Binongan",
+      "name": "Binongan Itneg",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "itd",
+      "inverted_name": "Tidung, Southern",
+      "name": "Southern Tidung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ite",
+      "name": "Itene",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "iti",
+      "inverted_name": "Itneg, Inlaod",
+      "name": "Inlaod Itneg",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "itk",
+      "name": "Judeo-Italian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "itl",
+      "name": "Itelmen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "itm",
+      "name": "Itu Mbon Uzo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ito",
+      "name": "Itonama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "itr",
+      "name": "Iteri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "its",
+      "name": "Isekiri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "itt",
+      "inverted_name": "Itneg, Maeng",
+      "name": "Maeng Itneg",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "itv",
+      "name": "Itawit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "itw",
+      "name": "Ito",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "itx",
+      "name": "Itik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ity",
+      "inverted_name": "Itneg, Moyadan",
+      "name": "Moyadan Itneg",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "itz",
+      "name": "Itzá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ium",
+      "inverted_name": "Mien, Iu",
+      "name": "Iu Mien",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ivb",
+      "name": "Ibatan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ivv",
+      "name": "Ivatan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iwk",
+      "name": "I-Wak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iwm",
+      "name": "Iwam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iwo",
+      "name": "Iwur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iws",
+      "inverted_name": "Iwam, Sepik",
+      "name": "Sepik Iwam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ixc",
+      "name": "Ixcatec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ixl",
+      "name": "Ixil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iya",
+      "name": "Iyayu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iyo",
+      "name": "Mesaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "iyx",
+      "name": "Yaka (Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "izh",
+      "name": "Ingrian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "izr",
+      "name": "Izere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "izz",
+      "name": "Izii",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jaa",
+      "name": "Jamamadí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jab",
+      "name": "Hyam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jac",
+      "name": "Popti'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jad",
+      "name": "Jahanka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jae",
+      "name": "Yabem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jaf",
+      "name": "Jara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jah",
+      "name": "Jah Hut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jaj",
+      "name": "Zazao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jak",
+      "name": "Jakun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jal",
+      "name": "Yalahatan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jam",
+      "inverted_name": "Creole English, Jamaican",
+      "name": "Jamaican Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jan",
+      "name": "Jandai",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "jao",
+      "name": "Yanyuwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jaq",
+      "name": "Yaqay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jas",
+      "inverted_name": "Javanese, New Caledonian",
+      "name": "New Caledonian Javanese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jat",
+      "name": "Jakati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jau",
+      "name": "Yaur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "jv",
+      "alpha_3": "jav",
+      "name": "Javanese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jax",
+      "inverted_name": "Malay, Jambi",
+      "name": "Jambi Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jay",
+      "name": "Yan-nhangu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jaz",
+      "name": "Jawe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jbe",
+      "name": "Judeo-Berber",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jbi",
+      "name": "Badjiri",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "jbj",
+      "name": "Arandai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jbk",
+      "name": "Barikewa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jbm",
+      "name": "Bijim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jbn",
+      "name": "Nafusi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jbo",
+      "name": "Lojban",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "jbr",
+      "name": "Jofotek-Bromnya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jbt",
+      "name": "Jabutí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jbu",
+      "name": "Jukun Takum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jbw",
+      "name": "Yawijibaya",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "jcs",
+      "name": "Jamaican Country Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jct",
+      "name": "Krymchak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jda",
+      "name": "Jad",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jdg",
+      "name": "Jadgali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jdt",
+      "name": "Judeo-Tat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jeb",
+      "name": "Jebero",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jee",
+      "name": "Jerung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jeh",
+      "name": "Jeh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jei",
+      "name": "Yei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jek",
+      "name": "Jeri Kuo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jel",
+      "name": "Yelmek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jen",
+      "name": "Dza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jer",
+      "name": "Jere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jet",
+      "name": "Manem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jeu",
+      "name": "Jonkor Bourmataguil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jgb",
+      "name": "Ngbee",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "jge",
+      "name": "Judeo-Georgian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jgk",
+      "name": "Gwak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jgo",
+      "name": "Ngomba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jhi",
+      "name": "Jehai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jhs",
+      "name": "Jhankot Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jia",
+      "name": "Jina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jib",
+      "name": "Jibu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jic",
+      "name": "Tol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jid",
+      "name": "Bu (Kaduna State)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jie",
+      "name": "Jilbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jig",
+      "name": "Jingulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jih",
+      "name": "sTodsde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jii",
+      "name": "Jiiddu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jil",
+      "name": "Jilim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jim",
+      "name": "Jimi (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jio",
+      "name": "Jiamao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jiq",
+      "name": "Guanyinqiao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jit",
+      "name": "Jita",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jiu",
+      "inverted_name": "Jinuo, Youle",
+      "name": "Youle Jinuo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jiv",
+      "name": "Shuar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jiy",
+      "inverted_name": "Jinuo, Buyuan",
+      "name": "Buyuan Jinuo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jje",
+      "name": "Jejueo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jjr",
+      "name": "Bankal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jka",
+      "name": "Kaera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jkm",
+      "inverted_name": "Karen, Mobwa",
+      "name": "Mobwa Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jko",
+      "name": "Kubo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jkp",
+      "inverted_name": "Karen, Paku",
+      "name": "Paku Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jkr",
+      "name": "Koro (India)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jks",
+      "name": "Amami Koniya Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jku",
+      "name": "Labir",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jle",
+      "name": "Ngile",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jls",
+      "name": "Jamaican Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jma",
+      "name": "Dima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jmb",
+      "name": "Zumbun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jmc",
+      "name": "Machame",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jmd",
+      "name": "Yamdena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jmi",
+      "name": "Jimi (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jml",
+      "name": "Jumli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jmn",
+      "inverted_name": "Naga, Makuri",
+      "name": "Makuri Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jmr",
+      "name": "Kamara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jms",
+      "name": "Mashi (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jmw",
+      "name": "Mouwase",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jmx",
+      "inverted_name": "Mixtec, Western Juxtlahuaca",
+      "name": "Western Juxtlahuaca Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jna",
+      "name": "Jangshung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jnd",
+      "name": "Jandavra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jng",
+      "name": "Yangman",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "jni",
+      "name": "Janji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jnj",
+      "name": "Yemsa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jnl",
+      "name": "Rawat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jns",
+      "name": "Jaunsari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "job",
+      "name": "Joba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jod",
+      "name": "Wojenaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jog",
+      "name": "Jogi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jor",
+      "name": "Jorá",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "jos",
+      "name": "Jordanian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jow",
+      "name": "Jowulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jpa",
+      "inverted_name": "Aramaic, Jewish Palestinian",
+      "name": "Jewish Palestinian Aramaic",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_2": "ja",
+      "alpha_3": "jpn",
+      "name": "Japanese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jpr",
+      "name": "Judeo-Persian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jqr",
+      "name": "Jaqaru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jra",
+      "name": "Jarai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jrb",
+      "name": "Judeo-Arabic",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jrr",
+      "name": "Jiru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jrt",
+      "name": "Jakattoe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jru",
+      "name": "Japrería",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jsl",
+      "name": "Japanese Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jua",
+      "name": "Júma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jub",
+      "name": "Wannu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "juc",
+      "name": "Jurchen",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "jud",
+      "name": "Worodougou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "juh",
+      "name": "Hõne",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jui",
+      "name": "Ngadjuri",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "juk",
+      "name": "Wapan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jul",
+      "name": "Jirel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jum",
+      "name": "Jumjum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jun",
+      "name": "Juang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "juo",
+      "name": "Jiba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jup",
+      "name": "Hupdë",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jur",
+      "name": "Jurúna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jus",
+      "name": "Jumla Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jut",
+      "name": "Jutish",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "juu",
+      "name": "Ju",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "juw",
+      "name": "Wãpha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "juy",
+      "name": "Juray",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jvd",
+      "name": "Javindo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jvn",
+      "inverted_name": "Javanese, Caribbean",
+      "name": "Caribbean Javanese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jwi",
+      "name": "Jwira-Pepesa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jya",
+      "name": "Jiarong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jye",
+      "inverted_name": "Arabic, Judeo-Yemeni",
+      "name": "Judeo-Yemeni Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "jyy",
+      "name": "Jaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kaa",
+      "name": "Kara-Kalpak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kab",
+      "name": "Kabyle",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kac",
+      "name": "Kachin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kad",
+      "name": "Adara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kae",
+      "name": "Ketangalan",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kaf",
+      "name": "Katso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kag",
+      "name": "Kajaman",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kah",
+      "name": "Kara (Central African Republic)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kai",
+      "name": "Karekare",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kaj",
+      "name": "Jju",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kak",
+      "name": "Kalanguya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "kl",
+      "alpha_3": "kal",
+      "name": "Kalaallisut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kam",
+      "name": "Kamba (Kenya)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "kn",
+      "alpha_3": "kan",
+      "name": "Kannada",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kao",
+      "name": "Xaasongaxango",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kap",
+      "name": "Bezhta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kaq",
+      "name": "Capanahua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ks",
+      "alpha_3": "kas",
+      "name": "Kashmiri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ka",
+      "alpha_3": "kat",
+      "bibliographic": "geo",
+      "name": "Georgian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "kr",
+      "alpha_3": "kau",
+      "name": "Kanuri",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kav",
+      "name": "Katukína",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kaw",
+      "name": "Kawi",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "kax",
+      "name": "Kao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kay",
+      "name": "Kamayurá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "kk",
+      "alpha_3": "kaz",
+      "name": "Kazakh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kba",
+      "name": "Kalarko",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kbb",
+      "name": "Kaxuiâna",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kbc",
+      "name": "Kadiwéu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbd",
+      "name": "Kabardian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbe",
+      "name": "Kanju",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbg",
+      "name": "Khamba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbh",
+      "name": "Camsá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbi",
+      "name": "Kaptiau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbj",
+      "name": "Kari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbk",
+      "inverted_name": "Koiari, Grass",
+      "name": "Grass Koiari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbl",
+      "name": "Kanembu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbm",
+      "name": "Iwal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbn",
+      "name": "Kare (Central African Republic)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbo",
+      "name": "Keliko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbp",
+      "name": "Kabiyè",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbq",
+      "name": "Kamano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbr",
+      "name": "Kafa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbs",
+      "name": "Kande",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbt",
+      "name": "Abadi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbu",
+      "name": "Kabutra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbv",
+      "name": "Dera (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbw",
+      "name": "Kaiep",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbx",
+      "name": "Ap Ma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kby",
+      "inverted_name": "Kanuri, Manga",
+      "name": "Manga Kanuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kbz",
+      "name": "Duhwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kca",
+      "name": "Khanty",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcb",
+      "name": "Kawacha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcc",
+      "name": "Lubila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcd",
+      "inverted_name": "Kanum, Ngkâlmpw",
+      "name": "Ngkâlmpw Kanum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kce",
+      "name": "Kaivi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcf",
+      "name": "Ukaan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcg",
+      "name": "Tyap",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kch",
+      "name": "Vono",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kci",
+      "name": "Kamantan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcj",
+      "name": "Kobiana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kck",
+      "name": "Kalanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcl",
+      "name": "Kela (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcm",
+      "name": "Gula (Central African Republic)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcn",
+      "name": "Nubi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kco",
+      "name": "Kinalakna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcp",
+      "name": "Kanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcq",
+      "name": "Kamo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcr",
+      "name": "Katla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcs",
+      "name": "Koenoem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kct",
+      "name": "Kaian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcu",
+      "name": "Kami (Tanzania)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcv",
+      "name": "Kete",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcw",
+      "name": "Kabwari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcx",
+      "name": "Kachama-Ganjule",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcy",
+      "name": "Korandje",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kcz",
+      "name": "Konongo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kda",
+      "name": "Worimi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kdc",
+      "name": "Kutu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdd",
+      "name": "Yankunytjatjara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kde",
+      "name": "Makonde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdf",
+      "name": "Mamusi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdg",
+      "name": "Seba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdh",
+      "name": "Tem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdi",
+      "name": "Kumam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdj",
+      "name": "Karamojong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdk",
+      "name": "Numèè",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdl",
+      "name": "Tsikimba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdm",
+      "name": "Kagoma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdn",
+      "name": "Kunda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdp",
+      "name": "Kaningdon-Nindem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdq",
+      "name": "Koch",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdr",
+      "name": "Karaim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdt",
+      "name": "Kuy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdu",
+      "name": "Kadaru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdw",
+      "name": "Koneraw",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdx",
+      "name": "Kam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdy",
+      "name": "Keder",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kdz",
+      "name": "Kwaja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kea",
+      "name": "Kabuverdianu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "keb",
+      "name": "Kélé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kec",
+      "name": "Keiga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ked",
+      "name": "Kerewe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kee",
+      "inverted_name": "Keres, Eastern",
+      "name": "Eastern Keres",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kef",
+      "name": "Kpessi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "keg",
+      "name": "Tese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "keh",
+      "name": "Keak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kei",
+      "name": "Kei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kej",
+      "name": "Kadar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kek",
+      "name": "Kekchí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kel",
+      "name": "Kela (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kem",
+      "name": "Kemak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ken",
+      "name": "Kenyang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "keo",
+      "name": "Kakwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kep",
+      "name": "Kaikadi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "keq",
+      "name": "Kamar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ker",
+      "name": "Kera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kes",
+      "name": "Kugbo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ket",
+      "name": "Ket",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "keu",
+      "name": "Akebu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kev",
+      "name": "Kanikkaran",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kew",
+      "inverted_name": "Kewa, West",
+      "name": "West Kewa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kex",
+      "name": "Kukna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "key",
+      "name": "Kupia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kez",
+      "name": "Kukele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfa",
+      "name": "Kodava",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfb",
+      "inverted_name": "Kolami, Northwestern",
+      "name": "Northwestern Kolami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfc",
+      "name": "Konda-Dora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfd",
+      "inverted_name": "Koraga, Korra",
+      "name": "Korra Koraga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfe",
+      "name": "Kota (India)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kff",
+      "name": "Koya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfg",
+      "name": "Kudiya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfh",
+      "name": "Kurichiya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfi",
+      "inverted_name": "Kurumba, Kannada",
+      "name": "Kannada Kurumba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfj",
+      "name": "Kemiehua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfk",
+      "name": "Kinnauri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfl",
+      "name": "Kung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfm",
+      "name": "Khunsari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfn",
+      "name": "Kuk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfo",
+      "name": "Koro (Côte d'Ivoire)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfp",
+      "name": "Korwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfq",
+      "name": "Korku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfr",
+      "name": "Kachhi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfs",
+      "name": "Bilaspuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kft",
+      "name": "Kanjari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfu",
+      "name": "Katkari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfv",
+      "name": "Kurmukar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfw",
+      "inverted_name": "Naga, Kharam",
+      "name": "Kharam Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfx",
+      "inverted_name": "Pahari, Kullu",
+      "name": "Kullu Pahari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfy",
+      "name": "Kumaoni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kfz",
+      "name": "Koromfé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kga",
+      "name": "Koyaga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgb",
+      "name": "Kawe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kge",
+      "name": "Komering",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgf",
+      "name": "Kube",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgg",
+      "name": "Kusunda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgi",
+      "name": "Selangor Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgj",
+      "inverted_name": "Kham, Gamale",
+      "name": "Gamale Kham",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgk",
+      "name": "Kaiwá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgl",
+      "name": "Kunggari",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kgm",
+      "name": "Karipúna",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kgn",
+      "name": "Karingani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgo",
+      "name": "Krongo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgp",
+      "name": "Kaingang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgq",
+      "name": "Kamoro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgr",
+      "name": "Abun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgs",
+      "name": "Kumbainggar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgt",
+      "name": "Somyev",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgu",
+      "name": "Kobol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgv",
+      "name": "Karas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgw",
+      "name": "Karon Dori",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgx",
+      "name": "Kamaru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kgy",
+      "name": "Kyerung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kha",
+      "name": "Khasi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khb",
+      "name": "Lü",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khc",
+      "name": "Tukang Besi North",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khd",
+      "inverted_name": "Kanum, Bädi",
+      "name": "Bädi Kanum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khe",
+      "name": "Korowai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khf",
+      "name": "Khuen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khg",
+      "inverted_name": "Tibetan, Khams",
+      "name": "Khams Tibetan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khh",
+      "name": "Kehu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khj",
+      "name": "Kuturmi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khk",
+      "inverted_name": "Mongolian, Halh",
+      "name": "Halh Mongolian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khl",
+      "name": "Lusi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "km",
+      "alpha_3": "khm",
+      "name": "Khmer",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khn",
+      "name": "Khandesi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kho",
+      "name": "Khotanese",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "khp",
+      "name": "Kapori",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khq",
+      "inverted_name": "Songhay, Koyra Chiini",
+      "name": "Koyra Chiini Songhay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khr",
+      "name": "Kharia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khs",
+      "name": "Kasua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kht",
+      "name": "Khamti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khu",
+      "name": "Nkhumbi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khv",
+      "name": "Khvarshi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khw",
+      "name": "Khowar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khx",
+      "name": "Kanu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khy",
+      "name": "Kele (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "khz",
+      "name": "Keapara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kia",
+      "name": "Kim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kib",
+      "name": "Koalib",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kic",
+      "name": "Kickapoo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kid",
+      "name": "Koshin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kie",
+      "name": "Kibet",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kif",
+      "inverted_name": "Kham, Eastern Parbate",
+      "name": "Eastern Parbate Kham",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kig",
+      "name": "Kimaama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kih",
+      "name": "Kilmeri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kii",
+      "name": "Kitsai",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kij",
+      "name": "Kilivila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ki",
+      "alpha_3": "kik",
+      "name": "Kikuyu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kil",
+      "name": "Kariya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kim",
+      "name": "Karagas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "rw",
+      "alpha_3": "kin",
+      "name": "Kinyarwanda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kio",
+      "name": "Kiowa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kip",
+      "inverted_name": "Kham, Sheshi",
+      "name": "Sheshi Kham",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kiq",
+      "name": "Kosadle",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ky",
+      "alpha_3": "kir",
+      "name": "Kirghiz",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kis",
+      "name": "Kis",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kit",
+      "name": "Agob",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kiu",
+      "name": "Kirmanjki (individual language)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kiv",
+      "name": "Kimbu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kiw",
+      "inverted_name": "Kiwai, Northeast",
+      "name": "Northeast Kiwai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kix",
+      "inverted_name": "Naga, Khiamniungan",
+      "name": "Khiamniungan Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kiy",
+      "name": "Kirikiri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kiz",
+      "name": "Kisi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kja",
+      "name": "Mlap",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjb",
+      "name": "Q'anjob'al",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjc",
+      "inverted_name": "Konjo, Coastal",
+      "name": "Coastal Konjo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjd",
+      "inverted_name": "Kiwai, Southern",
+      "name": "Southern Kiwai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kje",
+      "name": "Kisar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjg",
+      "name": "Khmu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjh",
+      "name": "Khakas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kji",
+      "name": "Zabana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjj",
+      "name": "Khinalugh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjk",
+      "inverted_name": "Konjo, Highland",
+      "name": "Highland Konjo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjl",
+      "inverted_name": "Kham, Western Parbate",
+      "name": "Western Parbate Kham",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjm",
+      "name": "Kháng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjn",
+      "name": "Kunjen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjo",
+      "inverted_name": "Kinnauri, Harijan",
+      "name": "Harijan Kinnauri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjp",
+      "inverted_name": "Karen, Pwo Eastern",
+      "name": "Pwo Eastern Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjq",
+      "inverted_name": "Keres, Western",
+      "name": "Western Keres",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjr",
+      "name": "Kurudu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjs",
+      "inverted_name": "Kewa, East",
+      "name": "East Kewa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjt",
+      "inverted_name": "Karen, Phrae Pwo",
+      "name": "Phrae Pwo Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kju",
+      "name": "Kashaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjv",
+      "name": "Kaikavian Literary Language",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "kjx",
+      "name": "Ramopa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjy",
+      "name": "Erave",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kjz",
+      "name": "Bumthangkha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kka",
+      "name": "Kakanda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkb",
+      "name": "Kwerisa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkc",
+      "name": "Odoodee",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkd",
+      "name": "Kinuku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kke",
+      "name": "Kakabe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkf",
+      "inverted_name": "Monpa, Kalaktang",
+      "name": "Kalaktang Monpa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkg",
+      "inverted_name": "Kalinga, Mabaka Valley",
+      "name": "Mabaka Valley Kalinga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkh",
+      "name": "Khün",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kki",
+      "name": "Kagulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkj",
+      "name": "Kako",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkk",
+      "name": "Kokota",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkl",
+      "inverted_name": "Yale, Kosarek",
+      "name": "Kosarek Yale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkm",
+      "name": "Kiong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkn",
+      "name": "Kon Keu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kko",
+      "name": "Karko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkp",
+      "name": "Gugubera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkq",
+      "name": "Kaeku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkr",
+      "name": "Kir-Balar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kks",
+      "name": "Giiwo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkt",
+      "name": "Koi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kku",
+      "name": "Tumi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkv",
+      "name": "Kangean",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkw",
+      "name": "Teke-Kukuya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkx",
+      "name": "Kohin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kky",
+      "name": "Guugu Yimidhirr",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kkz",
+      "name": "Kaska",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kla",
+      "name": "Klamath-Modoc",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "klb",
+      "name": "Kiliwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klc",
+      "name": "Kolbila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kld",
+      "name": "Gamilaraay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kle",
+      "name": "Kulung (Nepal)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klf",
+      "name": "Kendeje",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klg",
+      "name": "Tagakaulo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klh",
+      "name": "Weliki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kli",
+      "name": "Kalumpang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klj",
+      "name": "Khalaj",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klk",
+      "name": "Kono (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kll",
+      "inverted_name": "Kalagan, Kagan",
+      "name": "Kagan Kalagan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klm",
+      "name": "Migum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kln",
+      "name": "Kalenjin",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klo",
+      "name": "Kapya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klp",
+      "name": "Kamasa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klq",
+      "name": "Rumu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klr",
+      "name": "Khaling",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kls",
+      "name": "Kalasha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klt",
+      "name": "Nukna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klu",
+      "name": "Klao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klv",
+      "name": "Maskelynes",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klw",
+      "name": "Tado",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klx",
+      "name": "Koluwawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kly",
+      "name": "Kalao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "klz",
+      "name": "Kabola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kma",
+      "name": "Konni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmb",
+      "name": "Kimbundu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmc",
+      "inverted_name": "Dong, Southern",
+      "name": "Southern Dong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmd",
+      "inverted_name": "Kalinga, Majukayang",
+      "name": "Majukayang Kalinga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kme",
+      "name": "Bakole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmf",
+      "name": "Kare (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmg",
+      "name": "Kâte",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmh",
+      "name": "Kalam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmi",
+      "name": "Kami (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmj",
+      "name": "Kumarbhag Paharia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmk",
+      "inverted_name": "Kalinga, Limos",
+      "name": "Limos Kalinga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kml",
+      "inverted_name": "Kalinga, Tanudan",
+      "name": "Tanudan Kalinga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmm",
+      "name": "Kom (India)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmn",
+      "name": "Awtuw",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmo",
+      "name": "Kwoma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmp",
+      "name": "Gimme",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmq",
+      "name": "Kwama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmr",
+      "inverted_name": "Kurdish, Northern",
+      "name": "Northern Kurdish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kms",
+      "name": "Kamasau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmt",
+      "name": "Kemtuik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmu",
+      "name": "Kanite",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmv",
+      "inverted_name": "Creole French, Karipúna",
+      "name": "Karipúna Creole French",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmw",
+      "name": "Komo (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmx",
+      "name": "Waboda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmy",
+      "name": "Koma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kmz",
+      "name": "Khorasani Turkish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kna",
+      "name": "Dera (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knb",
+      "inverted_name": "Kalinga, Lubuagan",
+      "name": "Lubuagan Kalinga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knc",
+      "inverted_name": "Kanuri, Central",
+      "name": "Central Kanuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knd",
+      "name": "Konda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kne",
+      "name": "Kankanaey",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knf",
+      "name": "Mankanya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kng",
+      "name": "Koongo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kni",
+      "name": "Kanufi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knj",
+      "inverted_name": "Kanjobal, Western",
+      "name": "Western Kanjobal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knk",
+      "name": "Kuranko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knl",
+      "name": "Keninjal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knm",
+      "name": "Kanamarí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knn",
+      "name": "Konkani (individual language)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kno",
+      "name": "Kono (Sierra Leone)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knp",
+      "name": "Kwanja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knq",
+      "name": "Kintaq",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knr",
+      "name": "Kaningra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kns",
+      "name": "Kensiu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knt",
+      "inverted_name": "Katukína, Panoan",
+      "name": "Panoan Katukína",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knu",
+      "name": "Kono (Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knv",
+      "name": "Tabo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knw",
+      "name": "Kung-Ekoka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knx",
+      "name": "Kendayan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kny",
+      "name": "Kanyok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "knz",
+      "name": "Kalamsé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "koa",
+      "name": "Konomala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "koc",
+      "name": "Kpati",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kod",
+      "name": "Kodi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "koe",
+      "inverted_name": "Suri, Kacipo-Bale",
+      "name": "Kacipo-Bale Suri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kof",
+      "name": "Kubi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kog",
+      "name": "Cogui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "koh",
+      "name": "Koyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "koi",
+      "name": "Komi-Permyak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kok",
+      "name": "Konkani (macrolanguage)",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kol",
+      "name": "Kol (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "kv",
+      "alpha_3": "kom",
+      "name": "Komi",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_2": "kg",
+      "alpha_3": "kon",
+      "name": "Kongo",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "koo",
+      "name": "Konzo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kop",
+      "name": "Waube",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "koq",
+      "name": "Kota (Gabon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ko",
+      "alpha_3": "kor",
+      "name": "Korean",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kos",
+      "name": "Kosraean",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kot",
+      "name": "Lagwan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kou",
+      "name": "Koke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kov",
+      "name": "Kudu-Camo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kow",
+      "name": "Kugama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "koy",
+      "name": "Koyukon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "koz",
+      "name": "Korak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpa",
+      "name": "Kutto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpb",
+      "inverted_name": "Kurumba, Mullu",
+      "name": "Mullu Kurumba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpc",
+      "name": "Curripaco",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpd",
+      "name": "Koba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpe",
+      "name": "Kpelle",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpf",
+      "name": "Komba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpg",
+      "name": "Kapingamarangi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kph",
+      "name": "Kplang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpi",
+      "name": "Kofei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpj",
+      "name": "Karajá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpk",
+      "name": "Kpan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpl",
+      "name": "Kpala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpm",
+      "name": "Koho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpn",
+      "name": "Kepkiriwát",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kpo",
+      "name": "Ikposo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpq",
+      "name": "Korupun-Sela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpr",
+      "name": "Korafe-Yegha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kps",
+      "name": "Tehit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpt",
+      "name": "Karata",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpu",
+      "name": "Kafoa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpv",
+      "name": "Komi-Zyrian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpw",
+      "name": "Kobon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpx",
+      "inverted_name": "Koiali, Mountain",
+      "name": "Mountain Koiali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpy",
+      "name": "Koryak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kpz",
+      "name": "Kupsabiny",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqa",
+      "name": "Mum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqb",
+      "name": "Kovai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqc",
+      "name": "Doromu-Koki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqd",
+      "name": "Koy Sanjaq Surat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqe",
+      "name": "Kalagan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqf",
+      "name": "Kakabai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqg",
+      "name": "Khe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqh",
+      "name": "Kisankasa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqi",
+      "name": "Koitabu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqj",
+      "name": "Koromira",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqk",
+      "inverted_name": "Gbe, Kotafon",
+      "name": "Kotafon Gbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kql",
+      "name": "Kyenele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqm",
+      "name": "Khisa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqn",
+      "name": "Kaonde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqo",
+      "inverted_name": "Krahn, Eastern",
+      "name": "Eastern Krahn",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqp",
+      "name": "Kimré",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqq",
+      "name": "Krenak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqr",
+      "name": "Kimaragang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqs",
+      "inverted_name": "Kissi, Northern",
+      "name": "Northern Kissi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqt",
+      "inverted_name": "Kadazan, Klias River",
+      "name": "Klias River Kadazan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqu",
+      "name": "Seroa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kqv",
+      "name": "Okolod",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqw",
+      "name": "Kandas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqx",
+      "name": "Mser",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqy",
+      "name": "Koorete",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kqz",
+      "name": "Korana",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kra",
+      "name": "Kumhali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krb",
+      "name": "Karkin",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "krc",
+      "name": "Karachay-Balkar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krd",
+      "name": "Kairui-Midiki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kre",
+      "name": "Panará",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krf",
+      "name": "Koro (Vanuatu)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krh",
+      "name": "Kurama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kri",
+      "name": "Krio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krj",
+      "name": "Kinaray-A",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krk",
+      "name": "Kerek",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "krl",
+      "name": "Karelian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krn",
+      "name": "Sapo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krp",
+      "name": "Korop",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krr",
+      "name": "Krung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krs",
+      "name": "Gbaya (Sudan)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krt",
+      "inverted_name": "Kanuri, Tumari",
+      "name": "Tumari Kanuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kru",
+      "name": "Kurukh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krv",
+      "name": "Kavet",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krw",
+      "inverted_name": "Krahn, Western",
+      "name": "Western Krahn",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krx",
+      "name": "Karon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kry",
+      "name": "Kryts",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "krz",
+      "inverted_name": "Kanum, Sota",
+      "name": "Sota Kanum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksa",
+      "name": "Shuwa-Zamani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksb",
+      "name": "Shambala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksc",
+      "inverted_name": "Kalinga, Southern",
+      "name": "Southern Kalinga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksd",
+      "name": "Kuanua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kse",
+      "name": "Kuni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksf",
+      "name": "Bafia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksg",
+      "name": "Kusaghe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksh",
+      "name": "Kölsch",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksi",
+      "name": "Krisa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksj",
+      "name": "Uare",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksk",
+      "name": "Kansa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksl",
+      "name": "Kumalu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksm",
+      "name": "Kumba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksn",
+      "name": "Kasiguranin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kso",
+      "name": "Kofa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksp",
+      "name": "Kaba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksq",
+      "name": "Kwaami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksr",
+      "name": "Borong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kss",
+      "inverted_name": "Kisi, Southern",
+      "name": "Southern Kisi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kst",
+      "name": "Winyé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksu",
+      "name": "Khamyang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksv",
+      "name": "Kusu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksw",
+      "inverted_name": "Karen, S'gaw",
+      "name": "S'gaw Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksx",
+      "name": "Kedang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksy",
+      "name": "Kharia Thar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ksz",
+      "name": "Kodaku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kta",
+      "name": "Katua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktb",
+      "name": "Kambaata",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktc",
+      "name": "Kholok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktd",
+      "name": "Kokata",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kte",
+      "name": "Nubri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktf",
+      "name": "Kwami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktg",
+      "name": "Kalkutung",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kth",
+      "name": "Karanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kti",
+      "inverted_name": "Muyu, North",
+      "name": "North Muyu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktj",
+      "inverted_name": "Krumen, Plapo",
+      "name": "Plapo Krumen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktk",
+      "name": "Kaniet",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ktl",
+      "name": "Koroshi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktm",
+      "name": "Kurti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktn",
+      "name": "Karitiâna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kto",
+      "name": "Kuot",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktp",
+      "name": "Kaduo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktq",
+      "name": "Katabaga",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kts",
+      "inverted_name": "Muyu, South",
+      "name": "South Muyu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktt",
+      "name": "Ketum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktu",
+      "name": "Kituba (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktv",
+      "inverted_name": "Katu, Eastern",
+      "name": "Eastern Katu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktw",
+      "name": "Kato",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ktx",
+      "name": "Kaxararí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kty",
+      "name": "Kango (Bas-Uélé District)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ktz",
+      "name": "Juǀʼhoan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "kj",
+      "alpha_3": "kua",
+      "name": "Kuanyama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kub",
+      "name": "Kutep",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kuc",
+      "name": "Kwinsu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kud",
+      "name": "'Auhelawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kue",
+      "name": "Kuman (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kuf",
+      "inverted_name": "Katu, Western",
+      "name": "Western Katu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kug",
+      "name": "Kupa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kuh",
+      "name": "Kushi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kui",
+      "name": "Kuikúro-Kalapálo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kuj",
+      "name": "Kuria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kuk",
+      "name": "Kepo'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kul",
+      "name": "Kulere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kum",
+      "name": "Kumyk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kun",
+      "name": "Kunama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kuo",
+      "name": "Kumukio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kup",
+      "name": "Kunimaipa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kuq",
+      "name": "Karipuna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ku",
+      "alpha_3": "kur",
+      "name": "Kurdish",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kus",
+      "name": "Kusaal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kut",
+      "name": "Kutenai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kuu",
+      "inverted_name": "Kuskokwim, Upper",
+      "name": "Upper Kuskokwim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kuv",
+      "name": "Kur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kuw",
+      "name": "Kpagua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kux",
+      "name": "Kukatja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kuy",
+      "name": "Kuuku-Ya'u",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kuz",
+      "name": "Kunza",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kva",
+      "name": "Bagvalal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvb",
+      "name": "Kubu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvc",
+      "name": "Kove",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvd",
+      "name": "Kui (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kve",
+      "name": "Kalabakan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvf",
+      "name": "Kabalai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvg",
+      "name": "Kuni-Boazi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvh",
+      "name": "Komodo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvi",
+      "name": "Kwang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvj",
+      "name": "Psikye",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvk",
+      "name": "Korean Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvl",
+      "name": "Kayaw",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvm",
+      "name": "Kendem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvn",
+      "inverted_name": "Kuna, Border",
+      "name": "Border Kuna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvo",
+      "name": "Dobel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvp",
+      "name": "Kompane",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvq",
+      "inverted_name": "Karen, Geba",
+      "name": "Geba Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvr",
+      "name": "Kerinci",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvt",
+      "inverted_name": "Karen, Lahta",
+      "name": "Lahta Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvu",
+      "inverted_name": "Karen, Yinbaw",
+      "name": "Yinbaw Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvv",
+      "name": "Kola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvw",
+      "name": "Wersing",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvx",
+      "inverted_name": "Koli, Parkari",
+      "name": "Parkari Koli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvy",
+      "inverted_name": "Karen, Yintale",
+      "name": "Yintale Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kvz",
+      "name": "Tsakwambo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwa",
+      "name": "Dâw",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwb",
+      "name": "Kwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwc",
+      "name": "Likwala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwd",
+      "name": "Kwaio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwe",
+      "name": "Kwerba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwf",
+      "name": "Kwara'ae",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwg",
+      "name": "Sara Kaba Deme",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwh",
+      "name": "Kowiai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwi",
+      "name": "Awa-Cuaiquer",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwj",
+      "name": "Kwanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwk",
+      "name": "Kwakiutl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwl",
+      "name": "Kofyar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwm",
+      "name": "Kwambi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwn",
+      "name": "Kwangali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwo",
+      "name": "Kwomtari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwp",
+      "name": "Kodia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwr",
+      "name": "Kwer",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kws",
+      "name": "Kwese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwt",
+      "name": "Kwesten",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwu",
+      "name": "Kwakum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwv",
+      "name": "Sara Kaba Náà",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kww",
+      "name": "Kwinti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwx",
+      "name": "Khirwar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwy",
+      "inverted_name": "Kongo, San Salvador",
+      "name": "San Salvador Kongo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kwz",
+      "name": "Kwadi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kxa",
+      "name": "Kairiru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxb",
+      "name": "Krobu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxc",
+      "name": "Konso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxd",
+      "name": "Brunei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxf",
+      "inverted_name": "Karen, Manumanaw",
+      "name": "Manumanaw Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxh",
+      "name": "Karo (Ethiopia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxi",
+      "inverted_name": "Murut, Keningau",
+      "name": "Keningau Murut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxj",
+      "name": "Kulfa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxk",
+      "inverted_name": "Karen, Zayein",
+      "name": "Zayein Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxm",
+      "inverted_name": "Khmer, Northern",
+      "name": "Northern Khmer",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxn",
+      "inverted_name": "Melanau, Kanowit-Tanjong",
+      "name": "Kanowit-Tanjong Melanau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxo",
+      "name": "Kanoé",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kxp",
+      "inverted_name": "Koli, Wadiyara",
+      "name": "Wadiyara Koli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxq",
+      "inverted_name": "Kanum, Smärky",
+      "name": "Smärky Kanum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxr",
+      "name": "Koro (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxs",
+      "name": "Kangjia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxt",
+      "name": "Koiwat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxv",
+      "name": "Kuvi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxw",
+      "name": "Konai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxx",
+      "name": "Likuba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxy",
+      "name": "Kayong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kxz",
+      "name": "Kerewo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kya",
+      "name": "Kwaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyb",
+      "inverted_name": "Kalinga, Butbut",
+      "name": "Butbut Kalinga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyc",
+      "name": "Kyaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyd",
+      "name": "Karey",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kye",
+      "name": "Krache",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyf",
+      "name": "Kouya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyg",
+      "name": "Keyagana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyh",
+      "name": "Karok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyi",
+      "name": "Kiput",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyj",
+      "name": "Karao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyk",
+      "name": "Kamayo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyl",
+      "name": "Kalapuya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kym",
+      "name": "Kpatili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyn",
+      "inverted_name": "Binukidnon, Northern",
+      "name": "Northern Binukidnon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyo",
+      "name": "Kelon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyp",
+      "name": "Kang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyq",
+      "name": "Kenga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyr",
+      "name": "Kuruáya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kys",
+      "inverted_name": "Kayan, Baram",
+      "name": "Baram Kayan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyt",
+      "name": "Kayagar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyu",
+      "inverted_name": "Kayah, Western",
+      "name": "Western Kayah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyv",
+      "name": "Kayort",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyw",
+      "name": "Kudmali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyx",
+      "name": "Rapoisi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyy",
+      "name": "Kambaira",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kyz",
+      "name": "Kayabí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kza",
+      "inverted_name": "Karaboro, Western",
+      "name": "Western Karaboro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzb",
+      "name": "Kaibobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzc",
+      "inverted_name": "Kulango, Bondoukou",
+      "name": "Bondoukou Kulango",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzd",
+      "name": "Kadai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kze",
+      "name": "Kosena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzf",
+      "inverted_name": "Kaili, Da'a",
+      "name": "Da'a Kaili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzg",
+      "name": "Kikai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzi",
+      "name": "Kelabit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzk",
+      "name": "Kazukuru",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kzl",
+      "name": "Kayeli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzm",
+      "name": "Kais",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzn",
+      "name": "Kokola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzo",
+      "name": "Kaningi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzp",
+      "name": "Kaidipang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzq",
+      "name": "Kaike",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzr",
+      "name": "Karang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzs",
+      "inverted_name": "Dusun, Sugut",
+      "name": "Sugut Dusun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzu",
+      "name": "Kayupulau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzv",
+      "name": "Komyandaret",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzw",
+      "name": "Karirí-Xocó",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kzx",
+      "name": "Kamarian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "kzy",
+      "name": "Kango (Tshopo District)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "kzz",
+      "name": "Kalabra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "laa",
+      "inverted_name": "Subanen, Southern",
+      "name": "Southern Subanen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lab",
+      "name": "Linear A",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "lac",
+      "name": "Lacandon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lad",
+      "name": "Ladino",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lae",
+      "name": "Pattani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "laf",
+      "name": "Lafofa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lag",
+      "name": "Langi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lah",
+      "name": "Lahnda",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lai",
+      "name": "Lambya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "laj",
+      "name": "Lango (Uganda)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lal",
+      "name": "Lalia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lam",
+      "name": "Lamba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lan",
+      "name": "Laru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "lo",
+      "alpha_3": "lao",
+      "name": "Lao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lap",
+      "name": "Laka (Chad)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "laq",
+      "name": "Qabiao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lar",
+      "name": "Larteh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "las",
+      "name": "Lama (Togo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "la",
+      "alpha_3": "lat",
+      "name": "Latin",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "lau",
+      "name": "Laba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "lv",
+      "alpha_3": "lav",
+      "name": "Latvian",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "law",
+      "name": "Lauje",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lax",
+      "name": "Tiwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lay",
+      "inverted_name": "Bai, Lama",
+      "name": "Lama Bai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "laz",
+      "name": "Aribwatsa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "lbb",
+      "name": "Label",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbc",
+      "name": "Lakkia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbe",
+      "name": "Lak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbf",
+      "name": "Tinani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbg",
+      "name": "Laopang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbi",
+      "name": "La'bi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbj",
+      "name": "Ladakhi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbk",
+      "inverted_name": "Bontok, Central",
+      "name": "Central Bontok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbl",
+      "inverted_name": "Bikol, Libon",
+      "name": "Libon Bikol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbm",
+      "name": "Lodhi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbn",
+      "name": "Rmeet",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbo",
+      "name": "Laven",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbq",
+      "name": "Wampar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbr",
+      "name": "Lohorung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbs",
+      "name": "Libyan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbt",
+      "name": "Lachi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbu",
+      "name": "Labu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbv",
+      "name": "Lavatbura-Lamusong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbw",
+      "name": "Tolaki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lbx",
+      "name": "Lawangan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lby",
+      "name": "Lamalama",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "lbz",
+      "name": "Lardil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lcc",
+      "name": "Legenyem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lcd",
+      "name": "Lola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lce",
+      "name": "Loncong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lcf",
+      "name": "Lubu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lch",
+      "name": "Luchazi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lcl",
+      "name": "Lisela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lcm",
+      "name": "Tungag",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lcp",
+      "inverted_name": "Lawa, Western",
+      "name": "Western Lawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lcq",
+      "name": "Luhu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lcs",
+      "name": "Lisabata-Nuniali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lda",
+      "name": "Kla-Dan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ldb",
+      "name": "Dũya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ldd",
+      "name": "Luri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ldg",
+      "name": "Lenyima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ldh",
+      "name": "Lamja-Dengsa-Tola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ldi",
+      "name": "Laari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ldj",
+      "name": "Lemoro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ldk",
+      "name": "Leelau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ldl",
+      "name": "Kaan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ldm",
+      "name": "Landoma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ldn",
+      "name": "Láadan",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "ldo",
+      "name": "Loo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ldp",
+      "name": "Tso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ldq",
+      "name": "Lufu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lea",
+      "name": "Lega-Shabunda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "leb",
+      "name": "Lala-Bisa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lec",
+      "name": "Leco",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "led",
+      "name": "Lendu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lee",
+      "name": "Lyélé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lef",
+      "name": "Lelemi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "leh",
+      "name": "Lenje",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lei",
+      "name": "Lemio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lej",
+      "name": "Lengola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lek",
+      "name": "Leipon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lel",
+      "name": "Lele (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lem",
+      "name": "Nomaande",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "len",
+      "name": "Lenca",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "leo",
+      "name": "Leti (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lep",
+      "name": "Lepcha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "leq",
+      "name": "Lembena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ler",
+      "name": "Lenkau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "les",
+      "name": "Lese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "let",
+      "name": "Lesing-Gelimi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "leu",
+      "name": "Kara (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lev",
+      "name": "Lamma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lew",
+      "inverted_name": "Kaili, Ledo",
+      "name": "Ledo Kaili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lex",
+      "name": "Luang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ley",
+      "name": "Lemolang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lez",
+      "name": "Lezghian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lfa",
+      "name": "Lefa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lfn",
+      "name": "Lingua Franca Nova",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "lga",
+      "name": "Lungga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgb",
+      "name": "Laghu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgg",
+      "name": "Lugbara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgh",
+      "name": "Laghuu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgi",
+      "name": "Lengilu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgk",
+      "name": "Lingarak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgl",
+      "name": "Wala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgm",
+      "name": "Lega-Mwenga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgn",
+      "name": "T'apo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgo",
+      "name": "Lango (South Sudan)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgq",
+      "name": "Logba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgr",
+      "name": "Lengo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgt",
+      "name": "Pahi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgu",
+      "name": "Longgu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lgz",
+      "name": "Ligenza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lha",
+      "name": "Laha (Viet Nam)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lhh",
+      "name": "Laha (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lhi",
+      "name": "Lahu Shi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lhl",
+      "inverted_name": "Lohar, Lahul",
+      "name": "Lahul Lohar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lhm",
+      "name": "Lhomi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lhn",
+      "name": "Lahanan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lhp",
+      "name": "Lhokpu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lhs",
+      "name": "Mlahsö",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "lht",
+      "name": "Lo-Toga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lhu",
+      "name": "Lahu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lia",
+      "inverted_name": "Limba, West-Central",
+      "name": "West-Central Limba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lib",
+      "name": "Likum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lic",
+      "name": "Hlai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lid",
+      "name": "Nyindrou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lie",
+      "name": "Likila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lif",
+      "name": "Limbu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lig",
+      "name": "Ligbi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lih",
+      "name": "Lihir",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lij",
+      "name": "Ligurian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lik",
+      "name": "Lika",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lil",
+      "name": "Lillooet",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "li",
+      "alpha_3": "lim",
+      "name": "Limburgan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ln",
+      "alpha_3": "lin",
+      "name": "Lingala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lio",
+      "name": "Liki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lip",
+      "name": "Sekpele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "liq",
+      "name": "Libido",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lir",
+      "inverted_name": "English, Liberian",
+      "name": "Liberian English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lis",
+      "name": "Lisu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "lt",
+      "alpha_3": "lit",
+      "name": "Lithuanian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "liu",
+      "name": "Logorik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "liv",
+      "name": "Liv",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "liw",
+      "name": "Col",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lix",
+      "name": "Liabuku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "liy",
+      "name": "Banda-Bambari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "liz",
+      "name": "Libinza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lja",
+      "name": "Golpa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "lje",
+      "name": "Rampi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lji",
+      "name": "Laiyolo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ljl",
+      "name": "Li'o",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ljp",
+      "name": "Lampung Api",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ljw",
+      "name": "Yirandali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ljx",
+      "name": "Yuru",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "lka",
+      "name": "Lakalei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lkb",
+      "name": "Kabras",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lkc",
+      "name": "Kucong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lkd",
+      "name": "Lakondê",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lke",
+      "name": "Kenyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lkh",
+      "name": "Lakha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lki",
+      "name": "Laki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lkj",
+      "name": "Remun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lkl",
+      "name": "Laeko-Libuat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lkm",
+      "name": "Kalaamaya",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "lkn",
+      "name": "Lakon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lko",
+      "name": "Khayo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lkr",
+      "name": "Päri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lks",
+      "name": "Kisa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lkt",
+      "name": "Lakota",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lku",
+      "name": "Kungkari",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "lky",
+      "name": "Lokoya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lla",
+      "name": "Lala-Roba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "llb",
+      "name": "Lolo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "llc",
+      "name": "Lele (Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lld",
+      "name": "Ladin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lle",
+      "name": "Lele (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "llf",
+      "name": "Hermit",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "llg",
+      "name": "Lole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "llh",
+      "name": "Lamu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lli",
+      "name": "Teke-Laali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "llj",
+      "name": "Ladji Ladji",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "llk",
+      "name": "Lelak",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "lll",
+      "name": "Lilau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "llm",
+      "name": "Lasalimu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lln",
+      "name": "Lele (Chad)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "llp",
+      "inverted_name": "Efate, North",
+      "name": "North Efate",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "llq",
+      "name": "Lolak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lls",
+      "name": "Lithuanian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "llu",
+      "name": "Lau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "llx",
+      "name": "Lauan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lma",
+      "inverted_name": "Limba, East",
+      "name": "East Limba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmb",
+      "name": "Merei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmc",
+      "name": "Limilngan",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "lmd",
+      "name": "Lumun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lme",
+      "name": "Pévé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmf",
+      "inverted_name": "Lembata, South",
+      "name": "South Lembata",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmg",
+      "name": "Lamogai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmh",
+      "name": "Lambichhong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmi",
+      "name": "Lombi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmj",
+      "inverted_name": "Lembata, West",
+      "name": "West Lembata",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmk",
+      "name": "Lamkang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lml",
+      "name": "Hano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmn",
+      "name": "Lambadi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmo",
+      "name": "Lombard",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmp",
+      "name": "Limbum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmq",
+      "name": "Lamatuka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmr",
+      "name": "Lamalera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmu",
+      "name": "Lamenu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmv",
+      "name": "Lomaiviti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmw",
+      "inverted_name": "Miwok, Lake",
+      "name": "Lake Miwok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmx",
+      "name": "Laimbue",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lmy",
+      "name": "Lamboya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lna",
+      "name": "Langbashe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lnb",
+      "name": "Mbalanhu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lnd",
+      "name": "Lundayeh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lng",
+      "name": "Langobardic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "lnh",
+      "name": "Lanoh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lni",
+      "name": "Daantanai'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lnj",
+      "name": "Leningitij",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "lnl",
+      "inverted_name": "Banda, South Central",
+      "name": "South Central Banda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lnm",
+      "name": "Langam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lnn",
+      "name": "Lorediakarkar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lns",
+      "name": "Lamnso'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lnu",
+      "name": "Longuda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lnw",
+      "name": "Lanima",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "lnz",
+      "name": "Lonzo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "loa",
+      "name": "Loloda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lob",
+      "name": "Lobi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "loc",
+      "name": "Inonhan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "loe",
+      "name": "Saluan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lof",
+      "name": "Logol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "log",
+      "name": "Logo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "loh",
+      "name": "Narim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "loi",
+      "name": "Loma (Côte d'Ivoire)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "loj",
+      "name": "Lou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lok",
+      "name": "Loko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lol",
+      "name": "Mongo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lom",
+      "name": "Loma (Liberia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lon",
+      "inverted_name": "Lomwe, Malawi",
+      "name": "Malawi Lomwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "loo",
+      "name": "Lombo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lop",
+      "name": "Lopa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "loq",
+      "name": "Lobala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lor",
+      "name": "Téén",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "los",
+      "name": "Loniu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lot",
+      "name": "Otuho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lou",
+      "inverted_name": "Creole, Louisiana",
+      "name": "Louisiana Creole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lov",
+      "name": "Lopi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "low",
+      "inverted_name": "Lobu, Tampias",
+      "name": "Tampias Lobu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lox",
+      "name": "Loun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "loy",
+      "name": "Loke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "loz",
+      "name": "Lozi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lpa",
+      "name": "Lelepa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lpe",
+      "name": "Lepki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lpn",
+      "inverted_name": "Naga, Long Phuri",
+      "name": "Long Phuri Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lpo",
+      "name": "Lipo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lpx",
+      "name": "Lopit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lqr",
+      "name": "Logir",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lra",
+      "name": "Rara Bakati'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lrc",
+      "inverted_name": "Luri, Northern",
+      "name": "Northern Luri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lre",
+      "name": "Laurentian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "lrg",
+      "name": "Laragia",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "lri",
+      "name": "Marachi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lrk",
+      "name": "Loarki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lrl",
+      "name": "Lari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lrm",
+      "name": "Marama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lrn",
+      "name": "Lorang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lro",
+      "name": "Laro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lrr",
+      "inverted_name": "Yamphu, Southern",
+      "name": "Southern Yamphu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lrt",
+      "inverted_name": "Malay, Larantuka",
+      "name": "Larantuka Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lrv",
+      "name": "Larevat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lrz",
+      "name": "Lemerig",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsa",
+      "name": "Lasgerdi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsb",
+      "name": "Burundian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsc",
+      "name": "Albarradas Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsd",
+      "name": "Lishana Deni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lse",
+      "name": "Lusengo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsh",
+      "name": "Lish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsi",
+      "name": "Lashi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsl",
+      "name": "Latvian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsm",
+      "name": "Saamia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsn",
+      "name": "Tibetan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lso",
+      "name": "Laos Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsp",
+      "name": "Panamanian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsr",
+      "name": "Aruop",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lss",
+      "name": "Lasi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lst",
+      "name": "Trinidad and Tobago Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsv",
+      "name": "Sivia Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsw",
+      "name": "Seychelles Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lsy",
+      "name": "Mauritian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ltc",
+      "inverted_name": "Chinese, Late Middle",
+      "name": "Late Middle Chinese",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "ltg",
+      "name": "Latgalian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lth",
+      "name": "Thur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lti",
+      "name": "Leti (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ltn",
+      "name": "Latundê",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lto",
+      "name": "Tsotso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lts",
+      "name": "Tachoni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ltu",
+      "name": "Latu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "lb",
+      "alpha_3": "ltz",
+      "name": "Luxembourgish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lua",
+      "name": "Luba-Lulua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "lu",
+      "alpha_3": "lub",
+      "name": "Luba-Katanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "luc",
+      "name": "Aringa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lud",
+      "name": "Ludian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lue",
+      "name": "Luvale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "luf",
+      "name": "Laua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "lg",
+      "alpha_3": "lug",
+      "name": "Ganda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lui",
+      "name": "Luiseno",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "luj",
+      "name": "Luna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "luk",
+      "name": "Lunanakha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lul",
+      "name": "Olu'bo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lum",
+      "name": "Luimbi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lun",
+      "name": "Lunda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "luo",
+      "name": "Luo (Kenya and Tanzania)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lup",
+      "name": "Lumbu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "luq",
+      "name": "Lucumi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lur",
+      "name": "Laura",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lus",
+      "name": "Lushai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lut",
+      "name": "Lushootseed",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "luu",
+      "name": "Lumba-Yakkha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "luv",
+      "name": "Luwati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "luw",
+      "name": "Luo (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "luy",
+      "name": "Luyia",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "luz",
+      "inverted_name": "Luri, Southern",
+      "name": "Southern Luri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lva",
+      "name": "Maku'a",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lvi",
+      "name": "Lavi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lvk",
+      "name": "Lavukaleve",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lvs",
+      "inverted_name": "Latvian, Standard",
+      "name": "Standard Latvian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lvu",
+      "name": "Levuka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lwa",
+      "name": "Lwalu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lwe",
+      "name": "Lewo Eleng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lwg",
+      "name": "Wanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lwh",
+      "inverted_name": "Lachi, White",
+      "name": "White Lachi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lwl",
+      "inverted_name": "Lawa, Eastern",
+      "name": "Eastern Lawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lwm",
+      "name": "Laomian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lwo",
+      "name": "Luwo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lws",
+      "name": "Malawian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lwt",
+      "name": "Lewotobi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lwu",
+      "name": "Lawu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lww",
+      "name": "Lewo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lxm",
+      "name": "Lakurumau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lya",
+      "name": "Layakha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lyg",
+      "name": "Lyngngam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lyn",
+      "name": "Luyana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lzh",
+      "inverted_name": "Chinese, Literary",
+      "name": "Literary Chinese",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "lzl",
+      "name": "Litzlitz",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lzn",
+      "inverted_name": "Naga, Leinong",
+      "name": "Leinong Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "lzz",
+      "name": "Laz",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "maa",
+      "inverted_name": "Mazatec, San Jerónimo Tecóatl",
+      "name": "San Jerónimo Tecóatl Mazatec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mab",
+      "inverted_name": "Mixtec, Yutanduchi",
+      "name": "Yutanduchi Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mad",
+      "name": "Madurese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mae",
+      "name": "Bo-Rukul",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "maf",
+      "name": "Mafa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mag",
+      "name": "Magahi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "mh",
+      "alpha_3": "mah",
+      "name": "Marshallese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mai",
+      "name": "Maithili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "maj",
+      "inverted_name": "Mazatec, Jalapa De Díaz",
+      "name": "Jalapa De Díaz Mazatec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mak",
+      "name": "Makasar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ml",
+      "alpha_3": "mal",
+      "name": "Malayalam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mam",
+      "name": "Mam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "man",
+      "name": "Mandingo",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "maq",
+      "inverted_name": "Mazatec, Chiquihuitlán",
+      "name": "Chiquihuitlán Mazatec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "mr",
+      "alpha_3": "mar",
+      "name": "Marathi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mas",
+      "name": "Masai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mat",
+      "inverted_name": "Matlatzinca, San Francisco",
+      "name": "San Francisco Matlatzinca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mau",
+      "inverted_name": "Mazatec, Huautla",
+      "name": "Huautla Mazatec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mav",
+      "name": "Sateré-Mawé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "maw",
+      "name": "Mampruli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "max",
+      "inverted_name": "Malay, North Moluccan",
+      "name": "North Moluccan Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "maz",
+      "inverted_name": "Mazahua, Central",
+      "name": "Central Mazahua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mba",
+      "name": "Higaonon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbb",
+      "inverted_name": "Manobo, Western Bukidnon",
+      "name": "Western Bukidnon Manobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbc",
+      "name": "Macushi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbd",
+      "inverted_name": "Manobo, Dibabawon",
+      "name": "Dibabawon Manobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbe",
+      "name": "Molale",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mbf",
+      "inverted_name": "Malay, Baba",
+      "name": "Baba Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbh",
+      "name": "Mangseng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbi",
+      "inverted_name": "Manobo, Ilianen",
+      "name": "Ilianen Manobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbj",
+      "name": "Nadëb",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbk",
+      "name": "Malol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbl",
+      "name": "Maxakalí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbm",
+      "name": "Ombamba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbn",
+      "name": "Macaguán",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbo",
+      "name": "Mbo (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbp",
+      "name": "Malayo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbq",
+      "name": "Maisin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbr",
+      "name": "Nukak Makú",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbs",
+      "inverted_name": "Manobo, Sarangani",
+      "name": "Sarangani Manobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbt",
+      "inverted_name": "Manobo, Matigsalug",
+      "name": "Matigsalug Manobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbu",
+      "name": "Mbula-Bwazza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbv",
+      "name": "Mbulungish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbw",
+      "name": "Maring",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbx",
+      "name": "Mari (East Sepik Province)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mby",
+      "name": "Memoni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mbz",
+      "inverted_name": "Mixtec, Amoltepec",
+      "name": "Amoltepec Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mca",
+      "name": "Maca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcb",
+      "name": "Machiguenga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcc",
+      "name": "Bitur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcd",
+      "name": "Sharanahua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mce",
+      "inverted_name": "Mixtec, Itundujia",
+      "name": "Itundujia Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcf",
+      "name": "Matsés",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcg",
+      "name": "Mapoyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mch",
+      "name": "Maquiritari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mci",
+      "name": "Mese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcj",
+      "name": "Mvanip",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mck",
+      "name": "Mbunda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcl",
+      "name": "Macaguaje",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mcm",
+      "inverted_name": "Creole Portuguese, Malaccan",
+      "name": "Malaccan Creole Portuguese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcn",
+      "name": "Masana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mco",
+      "inverted_name": "Mixe, Coatlán",
+      "name": "Coatlán Mixe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcp",
+      "name": "Makaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcq",
+      "name": "Ese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcr",
+      "name": "Menya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcs",
+      "name": "Mambai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mct",
+      "name": "Mengisa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcu",
+      "inverted_name": "Mambila, Cameroon",
+      "name": "Cameroon Mambila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcv",
+      "name": "Minanibai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcw",
+      "name": "Mawa (Chad)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcx",
+      "name": "Mpiemo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcy",
+      "inverted_name": "Watut, South",
+      "name": "South Watut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mcz",
+      "name": "Mawan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mda",
+      "name": "Mada (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdb",
+      "name": "Morigi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdc",
+      "name": "Male (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdd",
+      "name": "Mbum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mde",
+      "name": "Maba (Chad)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdf",
+      "name": "Moksha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdg",
+      "name": "Massalat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdh",
+      "name": "Maguindanaon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdi",
+      "name": "Mamvu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdj",
+      "name": "Mangbetu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdk",
+      "name": "Mangbutu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdl",
+      "name": "Maltese Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdm",
+      "name": "Mayogo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdn",
+      "name": "Mbati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdp",
+      "name": "Mbala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdq",
+      "name": "Mbole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdr",
+      "name": "Mandar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mds",
+      "name": "Maria (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdt",
+      "name": "Mbere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdu",
+      "name": "Mboko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdv",
+      "inverted_name": "Mixtec, Santa Lucía Monteverde",
+      "name": "Santa Lucía Monteverde Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdw",
+      "name": "Mbosi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdx",
+      "name": "Dizin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdy",
+      "name": "Male (Ethiopia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mdz",
+      "name": "Suruí Do Pará",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mea",
+      "name": "Menka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "meb",
+      "name": "Ikobi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mec",
+      "name": "Marra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "med",
+      "name": "Melpa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mee",
+      "name": "Mengen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mef",
+      "name": "Megam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "meh",
+      "inverted_name": "Mixtec, Southwestern Tlaxiaco",
+      "name": "Southwestern Tlaxiaco Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mei",
+      "name": "Midob",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mej",
+      "name": "Meyah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mek",
+      "name": "Mekeo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mel",
+      "inverted_name": "Melanau, Central",
+      "name": "Central Melanau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mem",
+      "name": "Mangala",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "men",
+      "name": "Mende (Sierra Leone)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "meo",
+      "inverted_name": "Malay, Kedah",
+      "name": "Kedah Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mep",
+      "name": "Miriwoong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "meq",
+      "name": "Merey",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mer",
+      "name": "Meru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mes",
+      "name": "Masmaje",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "met",
+      "name": "Mato",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "meu",
+      "name": "Motu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mev",
+      "name": "Mano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mew",
+      "name": "Maaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mey",
+      "name": "Hassaniyya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mez",
+      "name": "Menominee",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfa",
+      "inverted_name": "Malay, Pattani",
+      "name": "Pattani Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfb",
+      "name": "Bangka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfc",
+      "name": "Mba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfd",
+      "name": "Mendankwe-Nkwen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfe",
+      "name": "Morisyen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mff",
+      "name": "Naki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfg",
+      "name": "Mogofin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfh",
+      "name": "Matal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfi",
+      "name": "Wandala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfj",
+      "name": "Mefele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfk",
+      "inverted_name": "Mofu, North",
+      "name": "North Mofu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfl",
+      "name": "Putai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfm",
+      "name": "Marghi South",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfn",
+      "inverted_name": "Mbembe, Cross River",
+      "name": "Cross River Mbembe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfo",
+      "name": "Mbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfp",
+      "inverted_name": "Malay, Makassar",
+      "name": "Makassar Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfq",
+      "name": "Moba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfr",
+      "name": "Marrithiyel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfs",
+      "name": "Mexican Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mft",
+      "name": "Mokerang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfu",
+      "name": "Mbwela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfv",
+      "name": "Mandjak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfw",
+      "name": "Mulaha",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mfx",
+      "name": "Melo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfy",
+      "name": "Mayo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mfz",
+      "name": "Mabaan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mga",
+      "inverted_name": "Irish, Middle (900-1200)",
+      "name": "Middle Irish (900-1200)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "mgb",
+      "name": "Mararit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgc",
+      "name": "Morokodo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgd",
+      "name": "Moru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mge",
+      "name": "Mango",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgf",
+      "name": "Maklew",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgg",
+      "name": "Mpumpong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgh",
+      "name": "Makhuwa-Meetto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgi",
+      "name": "Lijili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgj",
+      "name": "Abureni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgk",
+      "name": "Mawes",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgl",
+      "name": "Maleu-Kilenge",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgm",
+      "name": "Mambae",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgn",
+      "name": "Mbangi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgo",
+      "name": "Meta'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgp",
+      "inverted_name": "Magar, Eastern",
+      "name": "Eastern Magar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgq",
+      "name": "Malila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgr",
+      "name": "Mambwe-Lungu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgs",
+      "name": "Manda (Tanzania)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgt",
+      "name": "Mongol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgu",
+      "name": "Mailu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgv",
+      "name": "Matengo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgw",
+      "name": "Matumbi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgy",
+      "name": "Mbunga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mgz",
+      "name": "Mbugwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mha",
+      "name": "Manda (India)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhb",
+      "name": "Mahongwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhc",
+      "name": "Mocho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhd",
+      "name": "Mbugu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhe",
+      "name": "Besisi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhf",
+      "name": "Mamaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhg",
+      "name": "Margu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhi",
+      "name": "Ma'di",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhj",
+      "name": "Mogholi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhk",
+      "name": "Mungaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhl",
+      "name": "Mauwake",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhm",
+      "name": "Makhuwa-Moniga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhn",
+      "name": "Mócheno",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mho",
+      "name": "Mashi (Zambia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhp",
+      "inverted_name": "Malay, Balinese",
+      "name": "Balinese Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhq",
+      "name": "Mandan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhr",
+      "inverted_name": "Mari, Eastern",
+      "name": "Eastern Mari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhs",
+      "name": "Buru (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mht",
+      "name": "Mandahuaca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhu",
+      "name": "Digaro-Mishmi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhw",
+      "name": "Mbukushu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhx",
+      "name": "Maru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhy",
+      "name": "Ma'anyan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mhz",
+      "name": "Mor (Mor Islands)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mia",
+      "name": "Miami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mib",
+      "inverted_name": "Mixtec, Atatláhuca",
+      "name": "Atatláhuca Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mic",
+      "name": "Mi'kmaq",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mid",
+      "name": "Mandaic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mie",
+      "inverted_name": "Mixtec, Ocotepec",
+      "name": "Ocotepec Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mif",
+      "name": "Mofu-Gudur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mig",
+      "inverted_name": "Mixtec, San Miguel El Grande",
+      "name": "San Miguel El Grande Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mih",
+      "inverted_name": "Mixtec, Chayuco",
+      "name": "Chayuco Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mii",
+      "inverted_name": "Mixtec, Chigmecatitlán",
+      "name": "Chigmecatitlán Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mij",
+      "name": "Abar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mik",
+      "name": "Mikasuki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mil",
+      "inverted_name": "Mixtec, Peñoles",
+      "name": "Peñoles Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mim",
+      "inverted_name": "Mixtec, Alacatlatzala",
+      "name": "Alacatlatzala Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "min",
+      "name": "Minangkabau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mio",
+      "inverted_name": "Mixtec, Pinotepa Nacional",
+      "name": "Pinotepa Nacional Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mip",
+      "inverted_name": "Mixtec, Apasco-Apoala",
+      "name": "Apasco-Apoala Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "miq",
+      "name": "Mískito",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mir",
+      "inverted_name": "Mixe, Isthmus",
+      "name": "Isthmus Mixe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mis",
+      "name": "Uncoded languages",
+      "scope": "S",
+      "type": "S"
+    },
+    {
+      "alpha_3": "mit",
+      "inverted_name": "Mixtec, Southern Puebla",
+      "name": "Southern Puebla Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "miu",
+      "inverted_name": "Mixtec, Cacaloxtepec",
+      "name": "Cacaloxtepec Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "miw",
+      "name": "Akoye",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mix",
+      "inverted_name": "Mixtec, Mixtepec",
+      "name": "Mixtepec Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "miy",
+      "inverted_name": "Mixtec, Ayutla",
+      "name": "Ayutla Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "miz",
+      "inverted_name": "Mixtec, Coatzospan",
+      "name": "Coatzospan Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjb",
+      "name": "Makalero",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjc",
+      "inverted_name": "Mixtec, San Juan Colorado",
+      "name": "San Juan Colorado Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjd",
+      "inverted_name": "Maidu, Northwest",
+      "name": "Northwest Maidu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mje",
+      "name": "Muskum",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mjg",
+      "name": "Tu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjh",
+      "name": "Mwera (Nyasa)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mji",
+      "name": "Kim Mun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjj",
+      "name": "Mawak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjk",
+      "name": "Matukar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjl",
+      "name": "Mandeali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjm",
+      "name": "Medebur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjn",
+      "name": "Ma (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjo",
+      "name": "Malankuravan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjp",
+      "name": "Malapandaram",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjq",
+      "name": "Malaryan",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mjr",
+      "name": "Malavedan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjs",
+      "name": "Miship",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjt",
+      "name": "Sauria Paharia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mju",
+      "name": "Manna-Dora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjv",
+      "name": "Mannan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjw",
+      "name": "Karbi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjx",
+      "name": "Mahali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mjy",
+      "name": "Mahican",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mjz",
+      "name": "Majhi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mka",
+      "name": "Mbre",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkb",
+      "name": "Mal Paharia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkc",
+      "name": "Siliput",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "mk",
+      "alpha_3": "mkd",
+      "bibliographic": "mac",
+      "name": "Macedonian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mke",
+      "name": "Mawchi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkf",
+      "name": "Miya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkg",
+      "name": "Mak (China)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mki",
+      "name": "Dhatki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkj",
+      "name": "Mokilese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkk",
+      "name": "Byep",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkl",
+      "name": "Mokole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkm",
+      "name": "Moklen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkn",
+      "inverted_name": "Malay, Kupang",
+      "name": "Kupang Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mko",
+      "name": "Mingang Doso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkp",
+      "name": "Moikodi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkq",
+      "inverted_name": "Miwok, Bay",
+      "name": "Bay Miwok",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mkr",
+      "name": "Malas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mks",
+      "inverted_name": "Mixtec, Silacayoapan",
+      "name": "Silacayoapan Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkt",
+      "name": "Vamale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mku",
+      "inverted_name": "Maninka, Konyanka",
+      "name": "Konyanka Maninka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkv",
+      "name": "Mafea",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkw",
+      "name": "Kituba (Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkx",
+      "inverted_name": "Manobo, Kinamiging",
+      "name": "Kinamiging Manobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mky",
+      "inverted_name": "Makian, East",
+      "name": "East Makian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mkz",
+      "name": "Makasae",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mla",
+      "name": "Malo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlb",
+      "name": "Mbule",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlc",
+      "name": "Cao Lan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mle",
+      "name": "Manambu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlf",
+      "name": "Mal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "mg",
+      "alpha_3": "mlg",
+      "name": "Malagasy",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlh",
+      "name": "Mape",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mli",
+      "name": "Malimpung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlj",
+      "name": "Miltu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlk",
+      "name": "Ilwana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mll",
+      "name": "Malua Bay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlm",
+      "name": "Mulam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mln",
+      "name": "Malango",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlo",
+      "name": "Mlomp",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlp",
+      "name": "Bargam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlq",
+      "inverted_name": "Maninkakan, Western",
+      "name": "Western Maninkakan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlr",
+      "name": "Vame",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mls",
+      "name": "Masalit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "mt",
+      "alpha_3": "mlt",
+      "name": "Maltese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlu",
+      "name": "To'abaita",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlv",
+      "name": "Motlav",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlw",
+      "name": "Moloko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlx",
+      "name": "Malfaxal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mlz",
+      "name": "Malaynon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mma",
+      "name": "Mama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmb",
+      "name": "Momina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmc",
+      "inverted_name": "Mazahua, Michoacán",
+      "name": "Michoacán Mazahua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmd",
+      "name": "Maonan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mme",
+      "name": "Mae",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmf",
+      "name": "Mundat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmg",
+      "inverted_name": "Ambrym, North",
+      "name": "North Ambrym",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmh",
+      "name": "Mehináku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmi",
+      "name": "Musar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmj",
+      "name": "Majhwar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmk",
+      "name": "Mukha-Dora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mml",
+      "name": "Man Met",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmm",
+      "name": "Maii",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmn",
+      "name": "Mamanwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmo",
+      "inverted_name": "Buang, Mangga",
+      "name": "Mangga Buang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmp",
+      "name": "Siawi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmq",
+      "name": "Musak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmr",
+      "inverted_name": "Miao, Western Xiangxi",
+      "name": "Western Xiangxi Miao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmt",
+      "name": "Malalamai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmu",
+      "name": "Mmaala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmv",
+      "name": "Miriti",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mmw",
+      "name": "Emae",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmx",
+      "name": "Madak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmy",
+      "name": "Migaama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mmz",
+      "name": "Mabaale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mna",
+      "name": "Mbula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnb",
+      "name": "Muna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnc",
+      "name": "Manchu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnd",
+      "name": "Mondé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mne",
+      "name": "Naba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnf",
+      "name": "Mundani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mng",
+      "inverted_name": "Mnong, Eastern",
+      "name": "Eastern Mnong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnh",
+      "name": "Mono (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mni",
+      "name": "Manipuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnj",
+      "name": "Munji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnk",
+      "name": "Mandinka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnl",
+      "name": "Tiale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnm",
+      "name": "Mapena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnn",
+      "inverted_name": "Mnong, Southern",
+      "name": "Southern Mnong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnp",
+      "inverted_name": "Chinese, Min Bei",
+      "name": "Min Bei Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnq",
+      "name": "Minriq",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnr",
+      "name": "Mono (USA)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mns",
+      "name": "Mansi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnu",
+      "name": "Mer",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnv",
+      "name": "Rennell-Bellona",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnw",
+      "name": "Mon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnx",
+      "name": "Manikion",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mny",
+      "name": "Manyawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mnz",
+      "name": "Moni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "moa",
+      "name": "Mwan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "moc",
+      "name": "Mocoví",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mod",
+      "name": "Mobilian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "moe",
+      "name": "Innu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mog",
+      "name": "Mongondow",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "moh",
+      "name": "Mohawk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "moi",
+      "name": "Mboi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "moj",
+      "name": "Monzombo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mok",
+      "name": "Morori",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mom",
+      "name": "Mangue",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_2": "mn",
+      "alpha_3": "mon",
+      "name": "Mongolian",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "moo",
+      "name": "Monom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mop",
+      "name": "Mopán Maya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "moq",
+      "name": "Mor (Bomberai Peninsula)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mor",
+      "name": "Moro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mos",
+      "name": "Mossi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mot",
+      "name": "Barí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mou",
+      "name": "Mogum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mov",
+      "name": "Mohave",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mow",
+      "name": "Moi (Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mox",
+      "name": "Molima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "moy",
+      "name": "Shekkacho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "moz",
+      "name": "Mukulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpa",
+      "name": "Mpoto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpb",
+      "name": "Malak Malak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpc",
+      "name": "Mangarrayi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpd",
+      "name": "Machinere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpe",
+      "name": "Majang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpg",
+      "name": "Marba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mph",
+      "name": "Maung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpi",
+      "name": "Mpade",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpj",
+      "name": "Martu Wangka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpk",
+      "name": "Mbara (Chad)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpl",
+      "inverted_name": "Watut, Middle",
+      "name": "Middle Watut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpm",
+      "inverted_name": "Mixtec, Yosondúa",
+      "name": "Yosondúa Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpn",
+      "name": "Mindiri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpo",
+      "name": "Miu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpp",
+      "name": "Migabac",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpq",
+      "name": "Matís",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpr",
+      "name": "Vangunu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mps",
+      "name": "Dadibi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpt",
+      "name": "Mian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpu",
+      "name": "Makuráp",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpv",
+      "name": "Mungkip",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpw",
+      "name": "Mapidian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpx",
+      "name": "Misima-Panaeati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpy",
+      "name": "Mapia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mpz",
+      "name": "Mpi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqa",
+      "name": "Maba (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqb",
+      "name": "Mbuko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqc",
+      "name": "Mangole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqe",
+      "name": "Matepi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqf",
+      "name": "Momuna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqg",
+      "inverted_name": "Malay, Kota Bangun Kutai",
+      "name": "Kota Bangun Kutai Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqh",
+      "inverted_name": "Mixtec, Tlazoyaltepec",
+      "name": "Tlazoyaltepec Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqi",
+      "name": "Mariri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqj",
+      "name": "Mamasa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqk",
+      "inverted_name": "Manobo, Rajah Kabunsuwan",
+      "name": "Rajah Kabunsuwan Manobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mql",
+      "name": "Mbelime",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqm",
+      "inverted_name": "Marquesan, South",
+      "name": "South Marquesan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqn",
+      "name": "Moronene",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqo",
+      "name": "Modole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqp",
+      "name": "Manipa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqq",
+      "name": "Minokok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqr",
+      "name": "Mander",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqs",
+      "inverted_name": "Makian, West",
+      "name": "West Makian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqt",
+      "name": "Mok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqu",
+      "name": "Mandari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqv",
+      "name": "Mosimo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqw",
+      "name": "Murupi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqx",
+      "name": "Mamuju",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqy",
+      "name": "Manggarai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mqz",
+      "name": "Pano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mra",
+      "name": "Mlabri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrb",
+      "name": "Marino",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrc",
+      "name": "Maricopa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrd",
+      "inverted_name": "Magar, Western",
+      "name": "Western Magar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mre",
+      "name": "Martha's Vineyard Sign Language",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mrf",
+      "name": "Elseng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrg",
+      "name": "Mising",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrh",
+      "inverted_name": "Chin, Mara",
+      "name": "Mara Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "mi",
+      "alpha_3": "mri",
+      "bibliographic": "mao",
+      "name": "Maori",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrj",
+      "inverted_name": "Mari, Western",
+      "name": "Western Mari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrk",
+      "name": "Hmwaveke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrl",
+      "name": "Mortlockese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrm",
+      "name": "Merlav",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrn",
+      "name": "Cheke Holo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mro",
+      "name": "Mru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrp",
+      "name": "Morouas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrq",
+      "inverted_name": "Marquesan, North",
+      "name": "North Marquesan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrr",
+      "name": "Maria (India)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrs",
+      "name": "Maragus",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrt",
+      "name": "Marghi Central",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mru",
+      "name": "Mono (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrv",
+      "name": "Mangareva",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrw",
+      "name": "Maranao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrx",
+      "name": "Maremgi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mry",
+      "name": "Mandaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mrz",
+      "name": "Marind",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ms",
+      "alpha_3": "msa",
+      "bibliographic": "may",
+      "name": "Malay (macrolanguage)",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msb",
+      "name": "Masbatenyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msc",
+      "inverted_name": "Maninka, Sankaran",
+      "name": "Sankaran Maninka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msd",
+      "name": "Yucatec Maya Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mse",
+      "name": "Musey",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msf",
+      "name": "Mekwei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msg",
+      "name": "Moraid",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msh",
+      "inverted_name": "Malagasy, Masikoro",
+      "name": "Masikoro Malagasy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msi",
+      "inverted_name": "Malay, Sabah",
+      "name": "Sabah Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msj",
+      "name": "Ma (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msk",
+      "name": "Mansaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msl",
+      "name": "Molof",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msm",
+      "inverted_name": "Manobo, Agusan",
+      "name": "Agusan Manobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msn",
+      "name": "Vurës",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mso",
+      "name": "Mombum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msp",
+      "name": "Maritsauá",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "msq",
+      "name": "Caac",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msr",
+      "name": "Mongolian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mss",
+      "inverted_name": "Masela, West",
+      "name": "West Masela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msu",
+      "name": "Musom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msv",
+      "name": "Maslam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msw",
+      "name": "Mansoanka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msx",
+      "name": "Moresada",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msy",
+      "name": "Aruamu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "msz",
+      "name": "Momare",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mta",
+      "inverted_name": "Manobo, Cotabato",
+      "name": "Cotabato Manobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtb",
+      "name": "Anyin Morofo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtc",
+      "name": "Munit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtd",
+      "name": "Mualang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mte",
+      "name": "Mono (Solomon Islands)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtf",
+      "name": "Murik (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtg",
+      "name": "Una",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mth",
+      "name": "Munggui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mti",
+      "name": "Maiwa (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtj",
+      "name": "Moskona",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtk",
+      "name": "Mbe'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtl",
+      "name": "Montol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtm",
+      "name": "Mator",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mtn",
+      "name": "Matagalpa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mto",
+      "inverted_name": "Mixe, Totontepec",
+      "name": "Totontepec Mixe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtp",
+      "name": "Wichí Lhamtés Nocten",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtq",
+      "name": "Muong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtr",
+      "name": "Mewari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mts",
+      "name": "Yora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtt",
+      "name": "Mota",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtu",
+      "inverted_name": "Mixtec, Tututepec",
+      "name": "Tututepec Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtv",
+      "name": "Asaro'o",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtw",
+      "inverted_name": "Binukidnon, Southern",
+      "name": "Southern Binukidnon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mtx",
+      "inverted_name": "Mixtec, Tidaá",
+      "name": "Tidaá Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mty",
+      "name": "Nabi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mua",
+      "name": "Mundang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mub",
+      "name": "Mubi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "muc",
+      "name": "Ajumbu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mud",
+      "inverted_name": "Aleut, Mednyj",
+      "name": "Mednyj Aleut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mue",
+      "name": "Media Lengua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mug",
+      "name": "Musgu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "muh",
+      "name": "Mündü",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mui",
+      "name": "Musi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "muj",
+      "name": "Mabire",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "muk",
+      "name": "Mugom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mul",
+      "name": "Multiple languages",
+      "scope": "S",
+      "type": "S"
+    },
+    {
+      "alpha_3": "mum",
+      "name": "Maiwala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "muo",
+      "name": "Nyong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mup",
+      "name": "Malvi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "muq",
+      "inverted_name": "Miao, Eastern Xiangxi",
+      "name": "Eastern Xiangxi Miao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mur",
+      "name": "Murle",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mus",
+      "name": "Creek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mut",
+      "inverted_name": "Muria, Western",
+      "name": "Western Muria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "muu",
+      "name": "Yaaku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "muv",
+      "name": "Muthuvan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mux",
+      "name": "Bo-Ung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "muy",
+      "name": "Muyang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "muz",
+      "name": "Mursi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mva",
+      "name": "Manam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvb",
+      "name": "Mattole",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mvd",
+      "name": "Mamboru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mve",
+      "name": "Marwari (Pakistan)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvf",
+      "inverted_name": "Mongolian, Peripheral",
+      "name": "Peripheral Mongolian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvg",
+      "inverted_name": "Mixtec, Yucuañe",
+      "name": "Yucuañe Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvh",
+      "name": "Mulgi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvi",
+      "name": "Miyako",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvk",
+      "name": "Mekmek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvl",
+      "name": "Mbara (Australia)",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mvn",
+      "name": "Minaveha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvo",
+      "name": "Marovo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvp",
+      "name": "Duri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvq",
+      "name": "Moere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvr",
+      "name": "Marau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvs",
+      "name": "Massep",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvt",
+      "name": "Mpotovoro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvu",
+      "name": "Marfa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvv",
+      "inverted_name": "Murut, Tagal",
+      "name": "Tagal Murut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvw",
+      "name": "Machinga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvx",
+      "name": "Meoswar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvy",
+      "inverted_name": "Kohistani, Indus",
+      "name": "Indus Kohistani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mvz",
+      "name": "Mesqan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwa",
+      "name": "Mwatebu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwb",
+      "name": "Juwal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwc",
+      "name": "Are",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwe",
+      "name": "Mwera (Chimwera)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwf",
+      "name": "Murrinh-Patha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwg",
+      "name": "Aiklep",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwh",
+      "name": "Mouk-Aria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwi",
+      "name": "Labo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwk",
+      "inverted_name": "Maninkakan, Kita",
+      "name": "Kita Maninkakan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwl",
+      "name": "Mirandese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwm",
+      "name": "Sar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwn",
+      "name": "Nyamwanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwo",
+      "inverted_name": "Maewo, Central",
+      "name": "Central Maewo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwp",
+      "name": "Kala Lagaw Ya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwq",
+      "inverted_name": "Chin, Mün",
+      "name": "Mün Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwr",
+      "name": "Marwari",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mws",
+      "name": "Mwimbi-Muthambi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwt",
+      "name": "Moken",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwu",
+      "name": "Mittu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mwv",
+      "name": "Mentawai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mww",
+      "name": "Hmong Daw",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mwz",
+      "name": "Moingi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxa",
+      "inverted_name": "Mixtec, Northwest Oaxaca",
+      "name": "Northwest Oaxaca Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxb",
+      "inverted_name": "Mixtec, Tezoatlán",
+      "name": "Tezoatlán Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxc",
+      "name": "Manyika",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxd",
+      "name": "Modang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxe",
+      "name": "Mele-Fila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxf",
+      "name": "Malgbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxg",
+      "name": "Mbangala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxh",
+      "name": "Mvuba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxi",
+      "name": "Mozarabic",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "mxj",
+      "name": "Miju-Mishmi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxk",
+      "name": "Monumbo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxl",
+      "inverted_name": "Gbe, Maxi",
+      "name": "Maxi Gbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxm",
+      "name": "Meramera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxn",
+      "name": "Moi (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxo",
+      "name": "Mbowe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxp",
+      "inverted_name": "Mixe, Tlahuitoltepec",
+      "name": "Tlahuitoltepec Mixe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxq",
+      "inverted_name": "Mixe, Juquila",
+      "name": "Juquila Mixe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxr",
+      "name": "Murik (Malaysia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxs",
+      "inverted_name": "Mixtec, Huitepec",
+      "name": "Huitepec Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxt",
+      "inverted_name": "Mixtec, Jamiltepec",
+      "name": "Jamiltepec Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxu",
+      "name": "Mada (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxv",
+      "inverted_name": "Mixtec, Metlatónoc",
+      "name": "Metlatónoc Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxw",
+      "name": "Namo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxx",
+      "name": "Mahou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxy",
+      "inverted_name": "Mixtec, Southeastern Nochixtlán",
+      "name": "Southeastern Nochixtlán Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mxz",
+      "inverted_name": "Masela, Central",
+      "name": "Central Masela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "my",
+      "alpha_3": "mya",
+      "bibliographic": "bur",
+      "name": "Burmese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myb",
+      "name": "Mbay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myc",
+      "name": "Mayeka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mye",
+      "name": "Myene",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myf",
+      "name": "Bambassi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myg",
+      "name": "Manta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myh",
+      "name": "Makah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myj",
+      "name": "Mangayat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myk",
+      "inverted_name": "Senoufo, Mamara",
+      "name": "Mamara Senoufo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myl",
+      "name": "Moma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mym",
+      "name": "Me'en",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myo",
+      "name": "Anfillo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myp",
+      "name": "Pirahã",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myr",
+      "name": "Muniche",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mys",
+      "name": "Mesmes",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "myu",
+      "name": "Mundurukú",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myv",
+      "name": "Erzya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myw",
+      "name": "Muyuw",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myx",
+      "name": "Masaaba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myy",
+      "name": "Macuna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "myz",
+      "inverted_name": "Mandaic, Classical",
+      "name": "Classical Mandaic",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "mza",
+      "inverted_name": "Mixtec, Santa María Zacatepec",
+      "name": "Santa María Zacatepec Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzb",
+      "name": "Tumzabt",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzc",
+      "name": "Madagascar Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzd",
+      "name": "Malimba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mze",
+      "name": "Morawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzg",
+      "name": "Monastic Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzh",
+      "name": "Wichí Lhamtés Güisnay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzi",
+      "inverted_name": "Mazatec, Ixcatlán",
+      "name": "Ixcatlán Mazatec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzj",
+      "name": "Manya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzk",
+      "inverted_name": "Mambila, Nigeria",
+      "name": "Nigeria Mambila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzl",
+      "inverted_name": "Mixe, Mazatlán",
+      "name": "Mazatlán Mixe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzm",
+      "name": "Mumuye",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzn",
+      "name": "Mazanderani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzo",
+      "name": "Matipuhy",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "mzp",
+      "name": "Movima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzq",
+      "name": "Mori Atas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzr",
+      "name": "Marúbo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzs",
+      "name": "Macanese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzt",
+      "name": "Mintil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzu",
+      "name": "Inapang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzv",
+      "name": "Manza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzw",
+      "name": "Deg",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzx",
+      "name": "Mawayana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzy",
+      "name": "Mozambican Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "mzz",
+      "name": "Maiadomu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "naa",
+      "name": "Namla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nab",
+      "inverted_name": "Nambikuára, Southern",
+      "name": "Southern Nambikuára",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nac",
+      "name": "Narak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nae",
+      "name": "Naka'ela",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "naf",
+      "name": "Nabak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nag",
+      "name": "Naga Pidgin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "naj",
+      "name": "Nalu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nak",
+      "name": "Nakanai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nal",
+      "name": "Nalik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nam",
+      "name": "Ngan'gityemerri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nan",
+      "inverted_name": "Chinese, Min Nan",
+      "name": "Min Nan Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nao",
+      "name": "Naaba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nap",
+      "name": "Neapolitan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "naq",
+      "name": "Khoekhoe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nar",
+      "name": "Iguta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nas",
+      "name": "Naasioi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nat",
+      "name": "Ca̱hungwa̱rya̱",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "na",
+      "alpha_3": "nau",
+      "name": "Nauru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "nv",
+      "alpha_3": "nav",
+      "name": "Navajo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "naw",
+      "name": "Nawuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nax",
+      "name": "Nakwi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nay",
+      "name": "Ngarrindjeri",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "naz",
+      "inverted_name": "Nahuatl, Coatepec",
+      "name": "Coatepec Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nba",
+      "name": "Nyemba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbb",
+      "name": "Ndoe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbc",
+      "inverted_name": "Naga, Chang",
+      "name": "Chang Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbd",
+      "name": "Ngbinda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbe",
+      "inverted_name": "Naga, Konyak",
+      "name": "Konyak Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbg",
+      "name": "Nagarchal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbh",
+      "name": "Ngamo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbi",
+      "inverted_name": "Naga, Mao",
+      "name": "Mao Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbj",
+      "name": "Ngarinyman",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbk",
+      "name": "Nake",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "nr",
+      "alpha_3": "nbl",
+      "inverted_name": "Ndebele, South",
+      "name": "South Ndebele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbm",
+      "name": "Ngbaka Ma'bo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbn",
+      "name": "Kuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbo",
+      "name": "Nkukoli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbp",
+      "name": "Nnam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbq",
+      "name": "Nggem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbr",
+      "name": "Numana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbs",
+      "name": "Namibian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbt",
+      "name": "Na",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbu",
+      "inverted_name": "Naga, Rongmei",
+      "name": "Rongmei Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbv",
+      "name": "Ngamambo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nbw",
+      "inverted_name": "Ngbandi, Southern",
+      "name": "Southern Ngbandi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nby",
+      "name": "Ningera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nca",
+      "name": "Iyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncb",
+      "inverted_name": "Nicobarese, Central",
+      "name": "Central Nicobarese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncc",
+      "name": "Ponam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncd",
+      "name": "Nachering",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nce",
+      "name": "Yale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncf",
+      "name": "Notsi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncg",
+      "name": "Nisga'a",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nch",
+      "inverted_name": "Nahuatl, Central Huasteca",
+      "name": "Central Huasteca Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nci",
+      "inverted_name": "Nahuatl, Classical",
+      "name": "Classical Nahuatl",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "ncj",
+      "inverted_name": "Nahuatl, Northern Puebla",
+      "name": "Northern Puebla Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nck",
+      "name": "Na-kara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncl",
+      "inverted_name": "Nahuatl, Michoacán",
+      "name": "Michoacán Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncm",
+      "name": "Nambo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncn",
+      "name": "Nauna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nco",
+      "name": "Sibe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncq",
+      "inverted_name": "Katang, Northern",
+      "name": "Northern Katang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncr",
+      "name": "Ncane",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncs",
+      "name": "Nicaraguan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nct",
+      "inverted_name": "Naga, Chothe",
+      "name": "Chothe Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncu",
+      "name": "Chumburung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncx",
+      "inverted_name": "Nahuatl, Central Puebla",
+      "name": "Central Puebla Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ncz",
+      "name": "Natchez",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nda",
+      "name": "Ndasa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndb",
+      "name": "Kenswei Nsei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndc",
+      "name": "Ndau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndd",
+      "name": "Nde-Nsele-Nta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "nd",
+      "alpha_3": "nde",
+      "inverted_name": "Ndebele, North",
+      "name": "North Ndebele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndf",
+      "name": "Nadruvian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "ndg",
+      "name": "Ndengereko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndh",
+      "name": "Ndali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndi",
+      "name": "Samba Leko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndj",
+      "name": "Ndamba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndk",
+      "name": "Ndaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndl",
+      "name": "Ndolo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndm",
+      "name": "Ndam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndn",
+      "name": "Ngundi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ng",
+      "alpha_3": "ndo",
+      "name": "Ndonga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndp",
+      "name": "Ndo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndq",
+      "name": "Ndombe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndr",
+      "name": "Ndoola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nds",
+      "inverted_name": "German, Low",
+      "name": "Low German",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndt",
+      "name": "Ndunga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndu",
+      "name": "Dugun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndv",
+      "name": "Ndut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndw",
+      "name": "Ndobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndx",
+      "name": "Nduga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndy",
+      "name": "Lutos",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ndz",
+      "name": "Ndogo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nea",
+      "inverted_name": "Ngad'a, Eastern",
+      "name": "Eastern Ngad'a",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "neb",
+      "name": "Toura (Côte d'Ivoire)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nec",
+      "name": "Nedebang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ned",
+      "name": "Nde-Gbite",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nee",
+      "name": "Nêlêmwa-Nixumwak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nef",
+      "name": "Nefamese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "neg",
+      "name": "Negidal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "neh",
+      "name": "Nyenkha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nei",
+      "inverted_name": "Hittite, Neo-",
+      "name": "Neo-Hittite",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "nej",
+      "name": "Neko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nek",
+      "name": "Neku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nem",
+      "name": "Nemi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nen",
+      "name": "Nengone",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "neo",
+      "name": "Ná-Meo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ne",
+      "alpha_3": "nep",
+      "name": "Nepali (macrolanguage)",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "neq",
+      "inverted_name": "Mixe, North Central",
+      "name": "North Central Mixe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ner",
+      "name": "Yahadian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nes",
+      "inverted_name": "Kinnauri, Bhoti",
+      "name": "Bhoti Kinnauri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "net",
+      "name": "Nete",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "neu",
+      "name": "Neo",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "nev",
+      "name": "Nyaheun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "new",
+      "name": "Newari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nex",
+      "name": "Neme",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ney",
+      "name": "Neyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nez",
+      "name": "Nez Perce",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nfa",
+      "name": "Dhao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nfd",
+      "name": "Ahwai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nfl",
+      "name": "Ayiwo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nfr",
+      "name": "Nafaanra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nfu",
+      "name": "Mfumte",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nga",
+      "name": "Ngbaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngb",
+      "inverted_name": "Ngbandi, Northern",
+      "name": "Northern Ngbandi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngc",
+      "name": "Ngombe (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngd",
+      "name": "Ngando (Central African Republic)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nge",
+      "name": "Ngemba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngg",
+      "name": "Ngbaka Manza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngh",
+      "name": "Nǁng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngi",
+      "name": "Ngizim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngj",
+      "name": "Ngie",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngk",
+      "name": "Dalabon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngl",
+      "name": "Lomwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngm",
+      "name": "Ngatik Men's Creole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngn",
+      "name": "Ngwo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngp",
+      "name": "Ngulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngq",
+      "name": "Ngurimi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngr",
+      "name": "Engdewu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngs",
+      "name": "Gvoko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngt",
+      "name": "Kriang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngu",
+      "inverted_name": "Nahuatl, Guerrero",
+      "name": "Guerrero Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngv",
+      "name": "Nagumi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ngw",
+      "name": "Ngwaba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngx",
+      "name": "Nggwahyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngy",
+      "name": "Tibea",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ngz",
+      "name": "Ngungwel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nha",
+      "name": "Nhanda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhb",
+      "name": "Beng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhc",
+      "inverted_name": "Nahuatl, Tabasco",
+      "name": "Tabasco Nahuatl",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nhd",
+      "name": "Chiripá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhe",
+      "inverted_name": "Nahuatl, Eastern Huasteca",
+      "name": "Eastern Huasteca Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhf",
+      "name": "Nhuwala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhg",
+      "inverted_name": "Nahuatl, Tetelcingo",
+      "name": "Tetelcingo Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhh",
+      "name": "Nahari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhi",
+      "inverted_name": "Nahuatl, Zacatlán-Ahuacatlán-Tepetzintla",
+      "name": "Zacatlán-Ahuacatlán-Tepetzintla Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhk",
+      "inverted_name": "Nahuatl, Isthmus-Cosoleacaque",
+      "name": "Isthmus-Cosoleacaque Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhm",
+      "inverted_name": "Nahuatl, Morelos",
+      "name": "Morelos Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhn",
+      "inverted_name": "Nahuatl, Central",
+      "name": "Central Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nho",
+      "name": "Takuu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhp",
+      "inverted_name": "Nahuatl, Isthmus-Pajapan",
+      "name": "Isthmus-Pajapan Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhq",
+      "inverted_name": "Nahuatl, Huaxcaleca",
+      "name": "Huaxcaleca Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhr",
+      "name": "Naro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nht",
+      "inverted_name": "Nahuatl, Ometepec",
+      "name": "Ometepec Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhu",
+      "name": "Noone",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhv",
+      "inverted_name": "Nahuatl, Temascaltepec",
+      "name": "Temascaltepec Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhw",
+      "inverted_name": "Nahuatl, Western Huasteca",
+      "name": "Western Huasteca Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhx",
+      "inverted_name": "Nahuatl, Isthmus-Mecayapan",
+      "name": "Isthmus-Mecayapan Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhy",
+      "inverted_name": "Nahuatl, Northern Oaxaca",
+      "name": "Northern Oaxaca Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nhz",
+      "inverted_name": "Nahuatl, Santa María La Alta",
+      "name": "Santa María La Alta Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nia",
+      "name": "Nias",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nib",
+      "name": "Nakame",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nid",
+      "name": "Ngandi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nie",
+      "name": "Niellim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nif",
+      "name": "Nek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nig",
+      "name": "Ngalakgan",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nih",
+      "name": "Nyiha (Tanzania)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nii",
+      "name": "Nii",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nij",
+      "name": "Ngaju",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nik",
+      "inverted_name": "Nicobarese, Southern",
+      "name": "Southern Nicobarese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nil",
+      "name": "Nila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nim",
+      "name": "Nilamba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nin",
+      "name": "Ninzo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nio",
+      "name": "Nganasan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "niq",
+      "name": "Nandi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nir",
+      "name": "Nimboran",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nis",
+      "name": "Nimi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nit",
+      "inverted_name": "Kolami, Southeastern",
+      "name": "Southeastern Kolami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "niu",
+      "name": "Niuean",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "niv",
+      "name": "Gilyak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "niw",
+      "name": "Nimo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nix",
+      "name": "Hema",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "niy",
+      "name": "Ngiti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "niz",
+      "name": "Ningil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nja",
+      "name": "Nzanyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njb",
+      "inverted_name": "Naga, Nocte",
+      "name": "Nocte Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njd",
+      "name": "Ndonde Hamba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njh",
+      "inverted_name": "Naga, Lotha",
+      "name": "Lotha Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nji",
+      "name": "Gudanji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njj",
+      "name": "Njen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njl",
+      "name": "Njalgulgule",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njm",
+      "inverted_name": "Naga, Angami",
+      "name": "Angami Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njn",
+      "inverted_name": "Naga, Liangmai",
+      "name": "Liangmai Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njo",
+      "inverted_name": "Naga, Ao",
+      "name": "Ao Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njr",
+      "name": "Njerep",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njs",
+      "name": "Nisa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njt",
+      "name": "Ndyuka-Trio Pidgin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nju",
+      "name": "Ngadjunmaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njx",
+      "name": "Kunyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njy",
+      "name": "Njyem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "njz",
+      "name": "Nyishi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nka",
+      "name": "Nkoya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkb",
+      "inverted_name": "Naga, Khoibu",
+      "name": "Khoibu Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkc",
+      "name": "Nkongho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkd",
+      "name": "Koireng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nke",
+      "name": "Duke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkf",
+      "inverted_name": "Naga, Inpui",
+      "name": "Inpui Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkg",
+      "name": "Nekgini",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkh",
+      "inverted_name": "Naga, Khezha",
+      "name": "Khezha Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nki",
+      "inverted_name": "Naga, Thangal",
+      "name": "Thangal Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkj",
+      "name": "Nakai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkk",
+      "name": "Nokuku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkm",
+      "name": "Namat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkn",
+      "name": "Nkangala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nko",
+      "name": "Nkonya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkp",
+      "name": "Niuatoputapu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nkq",
+      "name": "Nkami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkr",
+      "name": "Nukuoro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nks",
+      "inverted_name": "Asmat, North",
+      "name": "North Asmat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkt",
+      "name": "Nyika (Tanzania)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nku",
+      "inverted_name": "Kulango, Bouna",
+      "name": "Bouna Kulango",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkv",
+      "name": "Nyika (Malawi and Zambia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkw",
+      "name": "Nkutu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkx",
+      "name": "Nkoroo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nkz",
+      "name": "Nkari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nla",
+      "name": "Ngombale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nlc",
+      "name": "Nalca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "nl",
+      "alpha_3": "nld",
+      "bibliographic": "dut",
+      "name": "Dutch",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nle",
+      "inverted_name": "Nyala, East",
+      "name": "East Nyala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nlg",
+      "name": "Gela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nli",
+      "name": "Grangali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nlj",
+      "name": "Nyali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nlk",
+      "inverted_name": "Yali, Ninia",
+      "name": "Ninia Yali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nll",
+      "name": "Nihali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nlm",
+      "name": "Mankiyali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nlo",
+      "name": "Ngul",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nlq",
+      "inverted_name": "Naga, Lao",
+      "name": "Lao Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nlu",
+      "name": "Nchumbulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nlv",
+      "inverted_name": "Nahuatl, Orizaba",
+      "name": "Orizaba Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nlw",
+      "name": "Walangama",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nlx",
+      "name": "Nahali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nly",
+      "name": "Nyamal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nlz",
+      "name": "Nalögo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nma",
+      "inverted_name": "Naga, Maram",
+      "name": "Maram Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmb",
+      "inverted_name": "Nambas, Big",
+      "name": "Big Nambas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmc",
+      "name": "Ngam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmd",
+      "name": "Ndumu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nme",
+      "inverted_name": "Naga, Mzieme",
+      "name": "Mzieme Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmf",
+      "inverted_name": "Naga, Tangkhul (India)",
+      "name": "Tangkhul Naga (India)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmg",
+      "name": "Kwasio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmh",
+      "inverted_name": "Naga, Monsang",
+      "name": "Monsang Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmi",
+      "name": "Nyam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmj",
+      "name": "Ngombe (Central African Republic)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmk",
+      "name": "Namakura",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nml",
+      "name": "Ndemli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmm",
+      "name": "Manangba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmn",
+      "name": "ǃXóõ",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmo",
+      "inverted_name": "Naga, Moyon",
+      "name": "Moyon Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmp",
+      "name": "Nimanbur",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nmq",
+      "name": "Nambya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmr",
+      "name": "Nimbari",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nms",
+      "name": "Letemboi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmt",
+      "name": "Namonuito",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmu",
+      "inverted_name": "Maidu, Northeast",
+      "name": "Northeast Maidu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmv",
+      "name": "Ngamini",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nmw",
+      "name": "Nimoa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmx",
+      "name": "Nama (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmy",
+      "name": "Namuyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nmz",
+      "name": "Nawdm",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nna",
+      "name": "Nyangumarta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnb",
+      "name": "Nande",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnc",
+      "name": "Nancere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnd",
+      "inverted_name": "Ambae, West",
+      "name": "West Ambae",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nne",
+      "name": "Ngandyera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnf",
+      "name": "Ngaing",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nng",
+      "inverted_name": "Naga, Maring",
+      "name": "Maring Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnh",
+      "name": "Ngiemboon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nni",
+      "inverted_name": "Nuaulu, North",
+      "name": "North Nuaulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnj",
+      "name": "Nyangatom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnk",
+      "name": "Nankina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnl",
+      "inverted_name": "Naga, Northern Rengma",
+      "name": "Northern Rengma Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnm",
+      "name": "Namia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnn",
+      "name": "Ngete",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "nn",
+      "alpha_3": "nno",
+      "name": "Norwegian Nynorsk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnp",
+      "inverted_name": "Naga, Wancho",
+      "name": "Wancho Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnq",
+      "name": "Ngindo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnr",
+      "name": "Narungga",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nnt",
+      "name": "Nanticoke",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nnu",
+      "name": "Dwang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nnv",
+      "name": "Nugunu (Australia)",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nnw",
+      "inverted_name": "Nuni, Southern",
+      "name": "Southern Nuni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nny",
+      "name": "Nyangga",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nnz",
+      "name": "Nda'nda'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "noa",
+      "name": "Woun Meu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "nb",
+      "alpha_3": "nob",
+      "name": "Norwegian Bokmål",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "noc",
+      "name": "Nuk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nod",
+      "inverted_name": "Thai, Northern",
+      "name": "Northern Thai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "noe",
+      "name": "Nimadi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nof",
+      "name": "Nomane",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nog",
+      "name": "Nogai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "noh",
+      "name": "Nomu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "noi",
+      "name": "Noiri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "noj",
+      "name": "Nonuya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nok",
+      "name": "Nooksack",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nol",
+      "name": "Nomlaki",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nom",
+      "name": "Nocamán",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "non",
+      "inverted_name": "Norse, Old",
+      "name": "Old Norse",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "nop",
+      "name": "Numanggang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "noq",
+      "name": "Ngongo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "no",
+      "alpha_3": "nor",
+      "name": "Norwegian",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nos",
+      "inverted_name": "Nisu, Eastern",
+      "name": "Eastern Nisu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "not",
+      "name": "Nomatsiguenga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nou",
+      "name": "Ewage-Notu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nov",
+      "name": "Novial",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "now",
+      "name": "Nyambo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "noy",
+      "name": "Noy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "noz",
+      "name": "Nayi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "npa",
+      "name": "Nar Phu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "npb",
+      "name": "Nupbikha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "npg",
+      "inverted_name": "Naga, Ponyo-Gongwang",
+      "name": "Ponyo-Gongwang Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nph",
+      "inverted_name": "Naga, Phom",
+      "name": "Phom Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "npi",
+      "name": "Nepali (individual language)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "npl",
+      "inverted_name": "Nahuatl, Southeastern Puebla",
+      "name": "Southeastern Puebla Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "npn",
+      "name": "Mondropolon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "npo",
+      "inverted_name": "Naga, Pochuri",
+      "name": "Pochuri Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nps",
+      "name": "Nipsan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "npu",
+      "inverted_name": "Naga, Puimei",
+      "name": "Puimei Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "npx",
+      "name": "Noipx",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "npy",
+      "name": "Napu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nqg",
+      "inverted_name": "Nago, Southern",
+      "name": "Southern Nago",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nqk",
+      "inverted_name": "Ede Nago, Kura",
+      "name": "Kura Ede Nago",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nql",
+      "name": "Ngendelengo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nqm",
+      "name": "Ndom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nqn",
+      "name": "Nen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nqo",
+      "name": "N'Ko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nqq",
+      "inverted_name": "Naga, Kyan-Karyaw",
+      "name": "Kyan-Karyaw Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nqt",
+      "name": "Nteng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nqy",
+      "inverted_name": "Naga, Akyaung Ari",
+      "name": "Akyaung Ari Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nra",
+      "name": "Ngom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nrb",
+      "name": "Nara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nrc",
+      "name": "Noric",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "nre",
+      "inverted_name": "Naga, Southern Rengma",
+      "name": "Southern Rengma Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nrf",
+      "name": "Jèrriais",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nrg",
+      "name": "Narango",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nri",
+      "inverted_name": "Naga, Chokri",
+      "name": "Chokri Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nrk",
+      "name": "Ngarla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nrl",
+      "name": "Ngarluma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nrm",
+      "name": "Narom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nrn",
+      "name": "Norn",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nrp",
+      "inverted_name": "Picene, North",
+      "name": "North Picene",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "nrr",
+      "name": "Norra",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nrt",
+      "inverted_name": "Kalapuya, Northern",
+      "name": "Northern Kalapuya",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nru",
+      "name": "Narua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nrx",
+      "name": "Ngurmbur",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nrz",
+      "name": "Lala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsa",
+      "inverted_name": "Naga, Sangtam",
+      "name": "Sangtam Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsb",
+      "name": "Lower Nossob",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nsc",
+      "name": "Nshi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsd",
+      "inverted_name": "Nisu, Southern",
+      "name": "Southern Nisu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nse",
+      "name": "Nsenga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsf",
+      "inverted_name": "Nisu, Northwestern",
+      "name": "Northwestern Nisu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsg",
+      "name": "Ngasa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsh",
+      "name": "Ngoshie",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsi",
+      "name": "Nigerian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsk",
+      "name": "Naskapi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsl",
+      "name": "Norwegian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsm",
+      "inverted_name": "Naga, Sumi",
+      "name": "Sumi Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsn",
+      "name": "Nehan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nso",
+      "name": "Pedi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsp",
+      "name": "Nepalese Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsq",
+      "inverted_name": "Miwok, Northern Sierra",
+      "name": "Northern Sierra Miwok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsr",
+      "name": "Maritime Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nss",
+      "name": "Nali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nst",
+      "inverted_name": "Naga, Tase",
+      "name": "Tase Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsu",
+      "inverted_name": "Nahuatl, Sierra Negra",
+      "name": "Sierra Negra Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsv",
+      "inverted_name": "Nisu, Southwestern",
+      "name": "Southwestern Nisu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsw",
+      "name": "Navut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsx",
+      "name": "Nsongo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsy",
+      "name": "Nasal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nsz",
+      "name": "Nisenan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ntd",
+      "inverted_name": "Tidung, Northern",
+      "name": "Northern Tidung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nte",
+      "name": "Nathembo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ntg",
+      "name": "Ngantangarra",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nti",
+      "name": "Natioro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ntj",
+      "name": "Ngaanyatjarra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ntk",
+      "name": "Ikoma-Nata-Isenye",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ntm",
+      "name": "Nateni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nto",
+      "name": "Ntomba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ntp",
+      "inverted_name": "Tepehuan, Northern",
+      "name": "Northern Tepehuan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ntr",
+      "name": "Delo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ntu",
+      "name": "Natügu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ntw",
+      "name": "Nottoway",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ntx",
+      "inverted_name": "Naga, Tangkhul (Myanmar)",
+      "name": "Tangkhul Naga (Myanmar)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nty",
+      "name": "Mantsi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ntz",
+      "name": "Natanzi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nua",
+      "name": "Yuanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nuc",
+      "name": "Nukuini",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nud",
+      "name": "Ngala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nue",
+      "name": "Ngundu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nuf",
+      "name": "Nusu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nug",
+      "name": "Nungali",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nuh",
+      "name": "Ndunda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nui",
+      "name": "Ngumbi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nuj",
+      "name": "Nyole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nuk",
+      "name": "Nuu-chah-nulth",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nul",
+      "name": "Nusa Laut",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "num",
+      "name": "Niuafo'ou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nun",
+      "name": "Anong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nuo",
+      "name": "Nguôn",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nup",
+      "name": "Nupe-Nupe-Tako",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nuq",
+      "name": "Nukumanu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nur",
+      "name": "Nukuria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nus",
+      "name": "Nuer",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nut",
+      "name": "Nung (Viet Nam)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nuu",
+      "name": "Ngbundu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nuv",
+      "inverted_name": "Nuni, Northern",
+      "name": "Northern Nuni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nuw",
+      "name": "Nguluwan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nux",
+      "name": "Mehek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nuy",
+      "name": "Nunggubuyu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nuz",
+      "inverted_name": "Nahuatl, Tlamacazapa",
+      "name": "Tlamacazapa Nahuatl",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nvh",
+      "name": "Nasarian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nvm",
+      "name": "Namiae",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nvo",
+      "name": "Nyokon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nwa",
+      "name": "Nawathinehena",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nwb",
+      "name": "Nyabwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nwc",
+      "inverted_name": "Newari, Classical",
+      "name": "Classical Newari",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "nwe",
+      "name": "Ngwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nwg",
+      "name": "Ngayawung",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nwi",
+      "inverted_name": "Tanna, Southwest",
+      "name": "Southwest Tanna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nwm",
+      "name": "Nyamusa-Molo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nwo",
+      "name": "Nauo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nwr",
+      "name": "Nawaru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nww",
+      "name": "Ndwewe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nwx",
+      "inverted_name": "Newar, Middle",
+      "name": "Middle Newar",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "nwy",
+      "name": "Nottoway-Meherrin",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nxa",
+      "name": "Nauete",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nxd",
+      "name": "Ngando (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nxe",
+      "name": "Nage",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nxg",
+      "name": "Ngad'a",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nxi",
+      "name": "Nindi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nxk",
+      "inverted_name": "Naga, Koki",
+      "name": "Koki Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nxl",
+      "inverted_name": "Nuaulu, South",
+      "name": "South Nuaulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nxm",
+      "name": "Numidian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "nxn",
+      "name": "Ngawun",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nxo",
+      "name": "Ndambomo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nxq",
+      "name": "Naxi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nxr",
+      "name": "Ninggerum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nxx",
+      "name": "Nafri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ny",
+      "alpha_3": "nya",
+      "name": "Nyanja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyb",
+      "name": "Nyangbo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyc",
+      "name": "Nyanga-li",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyd",
+      "name": "Nyore",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nye",
+      "name": "Nyengo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyf",
+      "name": "Giryama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyg",
+      "name": "Nyindu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyh",
+      "name": "Nyikina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyi",
+      "name": "Ama (Sudan)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyj",
+      "name": "Nyanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyk",
+      "name": "Nyaneka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyl",
+      "name": "Nyeu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nym",
+      "name": "Nyamwezi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyn",
+      "name": "Nyankole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyo",
+      "name": "Nyoro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyp",
+      "name": "Nyang'i",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nyq",
+      "name": "Nayini",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyr",
+      "name": "Nyiha (Malawi)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nys",
+      "name": "Nyungar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyt",
+      "name": "Nyawaygi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nyu",
+      "name": "Nyungwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyv",
+      "name": "Nyulnyul",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nyw",
+      "name": "Nyaw",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nyx",
+      "name": "Nganyaywana",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "nyy",
+      "name": "Nyakyusa-Ngonde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nza",
+      "inverted_name": "Mbembe, Tigon",
+      "name": "Tigon Mbembe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nzb",
+      "name": "Njebi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nzd",
+      "name": "Nzadi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nzi",
+      "name": "Nzima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nzk",
+      "name": "Nzakara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nzm",
+      "inverted_name": "Naga, Zeme",
+      "name": "Zeme Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nzs",
+      "name": "New Zealand Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nzu",
+      "name": "Teke-Nzikou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nzy",
+      "name": "Nzakambay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "nzz",
+      "inverted_name": "Dogon, Nanga Dama",
+      "name": "Nanga Dama Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oaa",
+      "name": "Orok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oac",
+      "name": "Oroch",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oar",
+      "inverted_name": "Aramaic, Old (up to 700 BCE)",
+      "name": "Old Aramaic (up to 700 BCE)",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "oav",
+      "inverted_name": "Avar, Old",
+      "name": "Old Avar",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "obi",
+      "name": "Obispeño",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "obk",
+      "inverted_name": "Bontok, Southern",
+      "name": "Southern Bontok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "obl",
+      "name": "Oblo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "obm",
+      "name": "Moabite",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "obo",
+      "inverted_name": "Manobo, Obo",
+      "name": "Obo Manobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "obr",
+      "inverted_name": "Burmese, Old",
+      "name": "Old Burmese",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "obt",
+      "inverted_name": "Breton, Old",
+      "name": "Old Breton",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "obu",
+      "name": "Obulom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oca",
+      "name": "Ocaina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "och",
+      "inverted_name": "Chinese, Old",
+      "name": "Old Chinese",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_2": "oc",
+      "alpha_3": "oci",
+      "name": "Occitan (post 1500)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ocm",
+      "inverted_name": "Cham, Old",
+      "name": "Old Cham",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "oco",
+      "inverted_name": "Cornish, Old",
+      "name": "Old Cornish",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "ocu",
+      "inverted_name": "Matlatzinca, Atzingo",
+      "name": "Atzingo Matlatzinca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oda",
+      "name": "Odut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "odk",
+      "name": "Od",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "odt",
+      "inverted_name": "Dutch, Old",
+      "name": "Old Dutch",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "odu",
+      "name": "Odual",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ofo",
+      "name": "Ofo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ofs",
+      "inverted_name": "Frisian, Old",
+      "name": "Old Frisian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "ofu",
+      "name": "Efutop",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ogb",
+      "name": "Ogbia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ogc",
+      "name": "Ogbah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oge",
+      "inverted_name": "Georgian, Old",
+      "name": "Old Georgian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "ogg",
+      "name": "Ogbogolo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ogo",
+      "name": "Khana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ogu",
+      "name": "Ogbronuagum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oht",
+      "inverted_name": "Hittite, Old",
+      "name": "Old Hittite",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "ohu",
+      "inverted_name": "Hungarian, Old",
+      "name": "Old Hungarian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "oia",
+      "name": "Oirata",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oie",
+      "name": "Okolie",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oin",
+      "inverted_name": "One, Inebu",
+      "name": "Inebu One",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ojb",
+      "inverted_name": "Ojibwa, Northwestern",
+      "name": "Northwestern Ojibwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ojc",
+      "inverted_name": "Ojibwa, Central",
+      "name": "Central Ojibwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ojg",
+      "inverted_name": "Ojibwa, Eastern",
+      "name": "Eastern Ojibwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "oj",
+      "alpha_3": "oji",
+      "name": "Ojibwa",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ojp",
+      "inverted_name": "Japanese, Old",
+      "name": "Old Japanese",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "ojs",
+      "inverted_name": "Ojibwa, Severn",
+      "name": "Severn Ojibwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ojv",
+      "name": "Ontong Java",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ojw",
+      "inverted_name": "Ojibwa, Western",
+      "name": "Western Ojibwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oka",
+      "name": "Okanagan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "okb",
+      "name": "Okobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "okc",
+      "name": "Kobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "okd",
+      "name": "Okodia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oke",
+      "name": "Okpe (Southwestern Edo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "okg",
+      "name": "Koko Babangk",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "okh",
+      "name": "Koresh-e Rostam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oki",
+      "name": "Okiek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "okj",
+      "name": "Oko-Juwoi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "okk",
+      "inverted_name": "One, Kwamtim",
+      "name": "Kwamtim One",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "okl",
+      "inverted_name": "Kentish Sign Language, Old",
+      "name": "Old Kentish Sign Language",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "okm",
+      "inverted_name": "Korean, Middle (10th-16th cent.)",
+      "name": "Middle Korean (10th-16th cent.)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "okn",
+      "name": "Oki-No-Erabu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oko",
+      "inverted_name": "Korean, Old (3rd-9th cent.)",
+      "name": "Old Korean (3rd-9th cent.)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "okr",
+      "name": "Kirike",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oks",
+      "name": "Oko-Eni-Osayen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oku",
+      "name": "Oku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "okv",
+      "name": "Orokaiva",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "okx",
+      "name": "Okpe (Northwestern Edo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "okz",
+      "inverted_name": "Khmer, Old",
+      "name": "Old Khmer",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "ola",
+      "name": "Walungge",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "old",
+      "name": "Mochi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ole",
+      "name": "Olekha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "olk",
+      "name": "Olkol",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "olm",
+      "name": "Oloma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "olo",
+      "name": "Livvi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "olr",
+      "name": "Olrat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "olt",
+      "inverted_name": "Lithuanian, Old",
+      "name": "Old Lithuanian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "olu",
+      "name": "Kuvale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oma",
+      "name": "Omaha-Ponca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "omb",
+      "inverted_name": "Ambae, East",
+      "name": "East Ambae",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "omc",
+      "name": "Mochica",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "omg",
+      "name": "Omagua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "omi",
+      "name": "Omi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "omk",
+      "name": "Omok",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "oml",
+      "name": "Ombo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "omn",
+      "name": "Minoan",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "omo",
+      "name": "Utarmbung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "omp",
+      "inverted_name": "Manipuri, Old",
+      "name": "Old Manipuri",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "omr",
+      "inverted_name": "Marathi, Old",
+      "name": "Old Marathi",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "omt",
+      "name": "Omotik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "omu",
+      "name": "Omurano",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "omw",
+      "inverted_name": "Tairora, South",
+      "name": "South Tairora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "omx",
+      "inverted_name": "Mon, Old",
+      "name": "Old Mon",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "omy",
+      "inverted_name": "Malay, Old",
+      "name": "Old Malay",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "ona",
+      "name": "Ona",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "onb",
+      "name": "Lingao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "one",
+      "name": "Oneida",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ong",
+      "name": "Olo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oni",
+      "name": "Onin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "onj",
+      "name": "Onjob",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "onk",
+      "inverted_name": "One, Kabore",
+      "name": "Kabore One",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "onn",
+      "name": "Onobasulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ono",
+      "name": "Onondaga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "onp",
+      "name": "Sartang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "onr",
+      "inverted_name": "One, Northern",
+      "name": "Northern One",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ons",
+      "name": "Ono",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ont",
+      "name": "Ontenu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "onu",
+      "name": "Unua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "onw",
+      "inverted_name": "Nubian, Old",
+      "name": "Old Nubian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "onx",
+      "name": "Onin Based Pidgin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ood",
+      "name": "Tohono O'odham",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oog",
+      "name": "Ong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oon",
+      "name": "Önge",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oor",
+      "name": "Oorlams",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oos",
+      "inverted_name": "Ossetic, Old",
+      "name": "Old Ossetic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "opa",
+      "name": "Okpamheri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "opk",
+      "name": "Kopkaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "opm",
+      "name": "Oksapmin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "opo",
+      "name": "Opao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "opt",
+      "name": "Opata",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "opy",
+      "name": "Ofayé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ora",
+      "name": "Oroha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "orc",
+      "name": "Orma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ore",
+      "name": "Orejón",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "org",
+      "name": "Oring",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "orh",
+      "name": "Oroqen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "or",
+      "alpha_3": "ori",
+      "name": "Oriya (macrolanguage)",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_2": "om",
+      "alpha_3": "orm",
+      "name": "Oromo",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "orn",
+      "name": "Orang Kanaq",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oro",
+      "name": "Orokolo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "orr",
+      "name": "Oruma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ors",
+      "name": "Orang Seletar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ort",
+      "inverted_name": "Oriya, Adivasi",
+      "name": "Adivasi Oriya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oru",
+      "name": "Ormuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "orv",
+      "inverted_name": "Russian, Old",
+      "name": "Old Russian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "orw",
+      "name": "Oro Win",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "orx",
+      "name": "Oro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ory",
+      "name": "Odia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "orz",
+      "name": "Ormu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "osa",
+      "name": "Osage",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "osc",
+      "name": "Oscan",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "osi",
+      "name": "Osing",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "osn",
+      "inverted_name": "Sundanese, Old",
+      "name": "Old Sundanese",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "oso",
+      "name": "Ososo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "osp",
+      "inverted_name": "Spanish, Old",
+      "name": "Old Spanish",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_2": "os",
+      "alpha_3": "oss",
+      "name": "Ossetian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ost",
+      "name": "Osatu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "osu",
+      "inverted_name": "One, Southern",
+      "name": "Southern One",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "osx",
+      "inverted_name": "Saxon, Old",
+      "name": "Old Saxon",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "ota",
+      "inverted_name": "Turkish, Ottoman (1500-1928)",
+      "name": "Ottoman Turkish (1500-1928)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "otb",
+      "inverted_name": "Tibetan, Old",
+      "name": "Old Tibetan",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "otd",
+      "name": "Ot Danum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ote",
+      "inverted_name": "Otomi, Mezquital",
+      "name": "Mezquital Otomi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oti",
+      "name": "Oti",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "otk",
+      "inverted_name": "Turkish, Old",
+      "name": "Old Turkish",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "otl",
+      "inverted_name": "Otomi, Tilapa",
+      "name": "Tilapa Otomi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "otm",
+      "inverted_name": "Otomi, Eastern Highland",
+      "name": "Eastern Highland Otomi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "otn",
+      "inverted_name": "Otomi, Tenango",
+      "name": "Tenango Otomi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "otq",
+      "inverted_name": "Otomi, Querétaro",
+      "name": "Querétaro Otomi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "otr",
+      "name": "Otoro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ots",
+      "inverted_name": "Otomi, Estado de México",
+      "name": "Estado de México Otomi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ott",
+      "inverted_name": "Otomi, Temoaya",
+      "name": "Temoaya Otomi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "otu",
+      "name": "Otuke",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "otw",
+      "name": "Ottawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "otx",
+      "inverted_name": "Otomi, Texcatepec",
+      "name": "Texcatepec Otomi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oty",
+      "inverted_name": "Tamil, Old",
+      "name": "Old Tamil",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "otz",
+      "inverted_name": "Otomi, Ixtenco",
+      "name": "Ixtenco Otomi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oua",
+      "name": "Tagargrent",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oub",
+      "name": "Glio-Oubi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oue",
+      "name": "Oune",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oui",
+      "inverted_name": "Uighur, Old",
+      "name": "Old Uighur",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "oum",
+      "name": "Ouma",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ovd",
+      "name": "Elfdalian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "owi",
+      "name": "Owiniga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "owl",
+      "inverted_name": "Welsh, Old",
+      "name": "Old Welsh",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "oyb",
+      "name": "Oy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oyd",
+      "name": "Oyda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oym",
+      "name": "Wayampi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "oyy",
+      "name": "Oya'oya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ozm",
+      "name": "Koonzime",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pab",
+      "name": "Parecís",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pac",
+      "name": "Pacoh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pad",
+      "name": "Paumarí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pae",
+      "name": "Pagibete",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "paf",
+      "name": "Paranawát",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pag",
+      "name": "Pangasinan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pah",
+      "name": "Tenharim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pai",
+      "name": "Pe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pak",
+      "name": "Parakanã",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pal",
+      "name": "Pahlavi",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "pam",
+      "name": "Pampanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "pa",
+      "alpha_3": "pan",
+      "name": "Panjabi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pao",
+      "inverted_name": "Paiute, Northern",
+      "name": "Northern Paiute",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pap",
+      "name": "Papiamento",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "paq",
+      "name": "Parya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "par",
+      "name": "Panamint",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pas",
+      "name": "Papasena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pau",
+      "name": "Palauan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pav",
+      "name": "Pakaásnovos",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "paw",
+      "name": "Pawnee",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pax",
+      "name": "Pankararé",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pay",
+      "name": "Pech",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "paz",
+      "name": "Pankararú",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pbb",
+      "name": "Páez",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbc",
+      "name": "Patamona",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbe",
+      "inverted_name": "Popoloca, Mezontla",
+      "name": "Mezontla Popoloca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbf",
+      "inverted_name": "Popoloca, Coyotepec",
+      "name": "Coyotepec Popoloca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbg",
+      "name": "Paraujano",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pbh",
+      "name": "E'ñapa Woromaipu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbi",
+      "name": "Parkwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbl",
+      "name": "Mak (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbm",
+      "inverted_name": "Mazatec, Puebla",
+      "name": "Puebla Mazatec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbn",
+      "name": "Kpasam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbo",
+      "name": "Papel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbp",
+      "name": "Badyara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbr",
+      "name": "Pangwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbs",
+      "inverted_name": "Pame, Central",
+      "name": "Central Pame",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbt",
+      "inverted_name": "Pashto, Southern",
+      "name": "Southern Pashto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbu",
+      "inverted_name": "Pashto, Northern",
+      "name": "Northern Pashto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pbv",
+      "name": "Pnar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pby",
+      "name": "Pyu (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pca",
+      "inverted_name": "Popoloca, Santa Inés Ahuatempan",
+      "name": "Santa Inés Ahuatempan Popoloca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pcb",
+      "name": "Pear",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pcc",
+      "name": "Bouyei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pcd",
+      "name": "Picard",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pce",
+      "inverted_name": "Palaung, Ruching",
+      "name": "Ruching Palaung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pcf",
+      "name": "Paliyan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pcg",
+      "name": "Paniya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pch",
+      "name": "Pardhan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pci",
+      "name": "Duruwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pcj",
+      "name": "Parenga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pck",
+      "inverted_name": "Chin, Paite",
+      "name": "Paite Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pcl",
+      "name": "Pardhi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pcm",
+      "inverted_name": "Pidgin, Nigerian",
+      "name": "Nigerian Pidgin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pcn",
+      "name": "Piti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pcp",
+      "name": "Pacahuara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pcw",
+      "name": "Pyapun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pda",
+      "name": "Anam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pdc",
+      "inverted_name": "German, Pennsylvania",
+      "name": "Pennsylvania German",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pdi",
+      "name": "Pa Di",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pdn",
+      "name": "Podena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pdo",
+      "name": "Padoe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pdt",
+      "name": "Plautdietsch",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pdu",
+      "name": "Kayan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pea",
+      "inverted_name": "Indonesian, Peranakan",
+      "name": "Peranakan Indonesian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "peb",
+      "inverted_name": "Pomo, Eastern",
+      "name": "Eastern Pomo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ped",
+      "name": "Mala (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pee",
+      "name": "Taje",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pef",
+      "inverted_name": "Pomo, Northeastern",
+      "name": "Northeastern Pomo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "peg",
+      "name": "Pengo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "peh",
+      "name": "Bonan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pei",
+      "name": "Chichimeca-Jonaz",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pej",
+      "inverted_name": "Pomo, Northern",
+      "name": "Northern Pomo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pek",
+      "name": "Penchal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pel",
+      "name": "Pekal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pem",
+      "name": "Phende",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "peo",
+      "inverted_name": "Persian, Old (ca. 600-400 B.C.)",
+      "name": "Old Persian (ca. 600-400 B.C.)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "pep",
+      "name": "Kunja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "peq",
+      "inverted_name": "Pomo, Southern",
+      "name": "Southern Pomo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pes",
+      "inverted_name": "Persian, Iranian",
+      "name": "Iranian Persian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pev",
+      "name": "Pémono",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pex",
+      "name": "Petats",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pey",
+      "name": "Petjo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pez",
+      "inverted_name": "Penan, Eastern",
+      "name": "Eastern Penan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pfa",
+      "name": "Pááfang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pfe",
+      "name": "Pere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pfl",
+      "name": "Pfaelzisch",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pga",
+      "inverted_name": "Creole Arabic, Sudanese",
+      "name": "Sudanese Creole Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pgd",
+      "name": "Gāndhārī",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "pgg",
+      "name": "Pangwali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pgi",
+      "name": "Pagi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pgk",
+      "name": "Rerep",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pgl",
+      "inverted_name": "Irish, Primitive",
+      "name": "Primitive Irish",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "pgn",
+      "name": "Paelignian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "pgs",
+      "name": "Pangseng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pgu",
+      "name": "Pagu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pgz",
+      "name": "Papua New Guinean Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pha",
+      "name": "Pa-Hng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "phd",
+      "name": "Phudagi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "phg",
+      "name": "Phuong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "phh",
+      "name": "Phukha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "phj",
+      "name": "Pahari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "phk",
+      "name": "Phake",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "phl",
+      "name": "Phalura",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "phm",
+      "name": "Phimbi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "phn",
+      "name": "Phoenician",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "pho",
+      "name": "Phunoi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "phq",
+      "name": "Phana'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "phr",
+      "name": "Pahari-Potwari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pht",
+      "name": "Phu Thai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "phu",
+      "name": "Phuan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "phv",
+      "name": "Pahlavani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "phw",
+      "name": "Phangduwali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pia",
+      "name": "Pima Bajo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pib",
+      "name": "Yine",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pic",
+      "name": "Pinji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pid",
+      "name": "Piaroa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pie",
+      "name": "Piro",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pif",
+      "name": "Pingelapese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pig",
+      "name": "Pisabo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pih",
+      "name": "Pitcairn-Norfolk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pij",
+      "name": "Pijao",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pil",
+      "name": "Yom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pim",
+      "name": "Powhatan",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pin",
+      "name": "Piame",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pio",
+      "name": "Piapoco",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pip",
+      "name": "Pero",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pir",
+      "name": "Piratapuyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pis",
+      "name": "Pijin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pit",
+      "name": "Pitta Pitta",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "piu",
+      "name": "Pintupi-Luritja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "piv",
+      "name": "Pileni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "piw",
+      "name": "Pimbwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pix",
+      "name": "Piu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "piy",
+      "name": "Piya-Kwonci",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "piz",
+      "name": "Pije",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pjt",
+      "name": "Pitjantjatjara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pka",
+      "inverted_name": "Prākrit, Ardhamāgadhī",
+      "name": "Ardhamāgadhī Prākrit",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "pkb",
+      "name": "Pokomo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pkc",
+      "name": "Paekche",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "pkg",
+      "name": "Pak-Tong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pkh",
+      "name": "Pankhu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pkn",
+      "name": "Pakanha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pko",
+      "name": "Pökoot",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pkp",
+      "name": "Pukapuka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pkr",
+      "inverted_name": "Kurumba, Attapady",
+      "name": "Attapady Kurumba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pks",
+      "name": "Pakistan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pkt",
+      "name": "Maleng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pku",
+      "name": "Paku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pla",
+      "name": "Miani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "plb",
+      "name": "Polonombauk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "plc",
+      "inverted_name": "Palawano, Central",
+      "name": "Central Palawano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pld",
+      "name": "Polari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ple",
+      "name": "Palu'e",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "plg",
+      "name": "Pilagá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "plh",
+      "name": "Paulohi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "pi",
+      "alpha_3": "pli",
+      "name": "Pali",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "plj",
+      "name": "Polci",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "plk",
+      "inverted_name": "Shina, Kohistani",
+      "name": "Kohistani Shina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pll",
+      "inverted_name": "Palaung, Shwe",
+      "name": "Shwe Palaung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pln",
+      "name": "Palenquero",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "plo",
+      "inverted_name": "Popoluca, Oluta",
+      "name": "Oluta Popoluca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "plq",
+      "name": "Palaic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "plr",
+      "inverted_name": "Senoufo, Palaka",
+      "name": "Palaka Senoufo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pls",
+      "inverted_name": "Popoloca, San Marcos Tlacoyalco",
+      "name": "San Marcos Tlacoyalco Popoloca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "plt",
+      "inverted_name": "Malagasy, Plateau",
+      "name": "Plateau Malagasy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "plu",
+      "name": "Palikúr",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "plv",
+      "inverted_name": "Palawano, Southwest",
+      "name": "Southwest Palawano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "plw",
+      "inverted_name": "Palawano, Brooke's Point",
+      "name": "Brooke's Point Palawano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ply",
+      "name": "Bolyu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "plz",
+      "name": "Paluan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pma",
+      "name": "Paama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmb",
+      "name": "Pambia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmd",
+      "name": "Pallanganmiddang",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pme",
+      "name": "Pwaamei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmf",
+      "name": "Pamona",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmh",
+      "inverted_name": "Prākrit, Māhārāṣṭri",
+      "name": "Māhārāṣṭri Prākrit",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "pmi",
+      "inverted_name": "Pumi, Northern",
+      "name": "Northern Pumi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmj",
+      "inverted_name": "Pumi, Southern",
+      "name": "Southern Pumi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmk",
+      "name": "Pamlico",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pml",
+      "name": "Lingua Franca",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pmm",
+      "name": "Pomo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmn",
+      "name": "Pam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmo",
+      "name": "Pom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmq",
+      "inverted_name": "Pame, Northern",
+      "name": "Northern Pame",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmr",
+      "name": "Paynamar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pms",
+      "name": "Piemontese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmt",
+      "name": "Tuamotuan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmw",
+      "inverted_name": "Miwok, Plains",
+      "name": "Plains Miwok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmx",
+      "inverted_name": "Naga, Poumei",
+      "name": "Poumei Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmy",
+      "inverted_name": "Malay, Papuan",
+      "name": "Papuan Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pmz",
+      "inverted_name": "Pame, Southern",
+      "name": "Southern Pame",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pna",
+      "name": "Punan Bah-Biau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnb",
+      "inverted_name": "Panjabi, Western",
+      "name": "Western Panjabi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnc",
+      "name": "Pannei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnd",
+      "name": "Mpinda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pne",
+      "inverted_name": "Penan, Western",
+      "name": "Western Penan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "png",
+      "name": "Pangu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnh",
+      "name": "Penrhyn",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pni",
+      "name": "Aoheng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnj",
+      "name": "Pinjarup",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pnk",
+      "name": "Paunaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnl",
+      "name": "Paleni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnm",
+      "name": "Punan Batu 1",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnn",
+      "name": "Pinai-Hagahai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pno",
+      "name": "Panobo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pnp",
+      "name": "Pancana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnq",
+      "name": "Pana (Burkina Faso)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnr",
+      "name": "Panim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pns",
+      "name": "Ponosakan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnt",
+      "name": "Pontic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnu",
+      "inverted_name": "Bunu, Jiongnai",
+      "name": "Jiongnai Bunu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnv",
+      "name": "Pinigura",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnw",
+      "name": "Banyjima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnx",
+      "name": "Phong-Kniang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pny",
+      "name": "Pinyin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pnz",
+      "name": "Pana (Central African Republic)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "poc",
+      "name": "Poqomam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "poe",
+      "inverted_name": "Popoloca, San Juan Atzingo",
+      "name": "San Juan Atzingo Popoloca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pof",
+      "name": "Poke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pog",
+      "name": "Potiguára",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "poh",
+      "name": "Poqomchi'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "poi",
+      "inverted_name": "Popoluca, Highland",
+      "name": "Highland Popoluca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pok",
+      "name": "Pokangá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "pl",
+      "alpha_3": "pol",
+      "name": "Polish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pom",
+      "inverted_name": "Pomo, Southeastern",
+      "name": "Southeastern Pomo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pon",
+      "name": "Pohnpeian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "poo",
+      "inverted_name": "Pomo, Central",
+      "name": "Central Pomo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pop",
+      "name": "Pwapwâ",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "poq",
+      "inverted_name": "Popoluca, Texistepec",
+      "name": "Texistepec Popoluca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "pt",
+      "alpha_3": "por",
+      "name": "Portuguese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pos",
+      "inverted_name": "Popoluca, Sayula",
+      "name": "Sayula Popoluca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pot",
+      "name": "Potawatomi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pov",
+      "inverted_name": "Crioulo, Upper Guinea",
+      "name": "Upper Guinea Crioulo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pow",
+      "inverted_name": "Popoloca, San Felipe Otlaltepec",
+      "name": "San Felipe Otlaltepec Popoloca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pox",
+      "name": "Polabian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "poy",
+      "name": "Pogolo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ppe",
+      "name": "Papi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ppi",
+      "name": "Paipai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ppk",
+      "name": "Uma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ppl",
+      "name": "Pipil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ppm",
+      "name": "Papuma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ppn",
+      "name": "Papapana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ppo",
+      "name": "Folopa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ppp",
+      "name": "Pelende",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ppq",
+      "name": "Pei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pps",
+      "inverted_name": "Popoloca, San Luís Temalacayuca",
+      "name": "San Luís Temalacayuca Popoloca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ppt",
+      "name": "Pare",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ppu",
+      "name": "Papora",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pqa",
+      "name": "Pa'a",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pqm",
+      "name": "Malecite-Passamaquoddy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prc",
+      "name": "Parachi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prd",
+      "name": "Parsi-Dari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pre",
+      "name": "Principense",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prf",
+      "name": "Paranan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prg",
+      "name": "Prussian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prh",
+      "name": "Porohanon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pri",
+      "name": "Paicî",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prk",
+      "name": "Parauk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prl",
+      "name": "Peruvian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prm",
+      "name": "Kibiri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prn",
+      "name": "Prasuni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pro",
+      "inverted_name": "Provençal, Old (to 1500)",
+      "name": "Old Provençal (to 1500)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "prp",
+      "name": "Parsi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prq",
+      "name": "Ashéninka Perené",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prr",
+      "name": "Puri",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "prs",
+      "name": "Dari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prt",
+      "name": "Phai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pru",
+      "name": "Puragi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prw",
+      "name": "Parawen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prx",
+      "name": "Purik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "prz",
+      "name": "Providencia Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "psa",
+      "inverted_name": "Awyu, Asue",
+      "name": "Asue Awyu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "psc",
+      "name": "Iranian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "psd",
+      "name": "Plains Indian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pse",
+      "inverted_name": "Malay, Central",
+      "name": "Central Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "psg",
+      "name": "Penang Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "psh",
+      "inverted_name": "Pashai, Southwest",
+      "name": "Southwest Pashai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "psi",
+      "inverted_name": "Pashai, Southeast",
+      "name": "Southeast Pashai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "psl",
+      "name": "Puerto Rican Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "psm",
+      "name": "Pauserna",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "psn",
+      "name": "Panasuan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pso",
+      "name": "Polish Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "psp",
+      "name": "Philippine Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "psq",
+      "name": "Pasi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "psr",
+      "name": "Portuguese Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pss",
+      "name": "Kaulong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pst",
+      "inverted_name": "Pashto, Central",
+      "name": "Central Pashto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "psu",
+      "inverted_name": "Prākrit, Sauraseni",
+      "name": "Sauraseni Prākrit",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "psw",
+      "name": "Port Sandwich",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "psy",
+      "name": "Piscataway",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pta",
+      "name": "Pai Tavytera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pth",
+      "name": "Pataxó Hã-Ha-Hãe",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pti",
+      "name": "Pindiini",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ptn",
+      "name": "Patani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pto",
+      "name": "Zo'é",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ptp",
+      "name": "Patep",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ptq",
+      "name": "Pattapu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ptr",
+      "name": "Piamatsina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ptt",
+      "name": "Enrekang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ptu",
+      "name": "Bambam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ptv",
+      "name": "Port Vato",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ptw",
+      "name": "Pentlatch",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pty",
+      "name": "Pathiya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pua",
+      "inverted_name": "Purepecha, Western Highland",
+      "name": "Western Highland Purepecha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pub",
+      "name": "Purum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "puc",
+      "name": "Punan Merap",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pud",
+      "name": "Punan Aput",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pue",
+      "name": "Puelche",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "puf",
+      "name": "Punan Merah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pug",
+      "name": "Phuie",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pui",
+      "name": "Puinave",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "puj",
+      "name": "Punan Tubu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pum",
+      "name": "Puma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "puo",
+      "name": "Puoc",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pup",
+      "name": "Pulabu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "puq",
+      "name": "Puquina",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pur",
+      "name": "Puruborá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ps",
+      "alpha_3": "pus",
+      "name": "Pushto",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "put",
+      "name": "Putoh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "puu",
+      "name": "Punu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "puw",
+      "name": "Puluwatese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pux",
+      "name": "Puare",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "puy",
+      "name": "Purisimeño",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pwa",
+      "name": "Pawaia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pwb",
+      "name": "Panawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pwg",
+      "name": "Gapapaiwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pwi",
+      "name": "Patwin",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "pwm",
+      "name": "Molbog",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pwn",
+      "name": "Paiwan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pwo",
+      "inverted_name": "Karen, Pwo Western",
+      "name": "Pwo Western Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pwr",
+      "name": "Powari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pww",
+      "inverted_name": "Karen, Pwo Northern",
+      "name": "Pwo Northern Karen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pxm",
+      "inverted_name": "Mixe, Quetzaltepec",
+      "name": "Quetzaltepec Mixe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pye",
+      "inverted_name": "Krumen, Pye",
+      "name": "Pye Krumen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pym",
+      "name": "Fyam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pyn",
+      "name": "Poyanáwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pys",
+      "name": "Paraguayan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pyu",
+      "name": "Puyuma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pyx",
+      "name": "Pyu (Myanmar)",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "pyy",
+      "name": "Pyen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pzh",
+      "name": "Pazeh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "pzn",
+      "name": "Jejara Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qua",
+      "name": "Quapaw",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qub",
+      "inverted_name": "Quechua, Huallaga Huánuco",
+      "name": "Huallaga Huánuco Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "quc",
+      "name": "K'iche'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qud",
+      "inverted_name": "Quichua, Calderón Highland",
+      "name": "Calderón Highland Quichua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "qu",
+      "alpha_3": "que",
+      "name": "Quechua",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "quf",
+      "inverted_name": "Quechua, Lambayeque",
+      "name": "Lambayeque Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qug",
+      "inverted_name": "Quichua, Chimborazo Highland",
+      "name": "Chimborazo Highland Quichua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "quh",
+      "inverted_name": "Quechua, South Bolivian",
+      "name": "South Bolivian Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qui",
+      "name": "Quileute",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "quk",
+      "inverted_name": "Quechua, Chachapoyas",
+      "name": "Chachapoyas Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qul",
+      "inverted_name": "Quechua, North Bolivian",
+      "name": "North Bolivian Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qum",
+      "name": "Sipacapense",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qun",
+      "name": "Quinault",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "qup",
+      "inverted_name": "Quechua, Southern Pastaza",
+      "name": "Southern Pastaza Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "quq",
+      "name": "Quinqui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qur",
+      "inverted_name": "Quechua, Yanahuanca Pasco",
+      "name": "Yanahuanca Pasco Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qus",
+      "inverted_name": "Quichua, Santiago del Estero",
+      "name": "Santiago del Estero Quichua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "quv",
+      "name": "Sacapulteco",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "quw",
+      "inverted_name": "Quichua, Tena Lowland",
+      "name": "Tena Lowland Quichua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qux",
+      "inverted_name": "Quechua, Yauyos",
+      "name": "Yauyos Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "quy",
+      "inverted_name": "Quechua, Ayacucho",
+      "name": "Ayacucho Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "quz",
+      "inverted_name": "Quechua, Cusco",
+      "name": "Cusco Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qva",
+      "inverted_name": "Quechua, Ambo-Pasco",
+      "name": "Ambo-Pasco Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qvc",
+      "inverted_name": "Quechua, Cajamarca",
+      "name": "Cajamarca Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qve",
+      "inverted_name": "Quechua, Eastern Apurímac",
+      "name": "Eastern Apurímac Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qvh",
+      "inverted_name": "Quechua, Huamalíes-Dos de Mayo Huánuco",
+      "name": "Huamalíes-Dos de Mayo Huánuco Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qvi",
+      "inverted_name": "Quichua, Imbabura Highland",
+      "name": "Imbabura Highland Quichua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qvj",
+      "inverted_name": "Quichua, Loja Highland",
+      "name": "Loja Highland Quichua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qvl",
+      "inverted_name": "Quechua, Cajatambo North Lima",
+      "name": "Cajatambo North Lima Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qvm",
+      "inverted_name": "Quechua, Margos-Yarowilca-Lauricocha",
+      "name": "Margos-Yarowilca-Lauricocha Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qvn",
+      "inverted_name": "Quechua, North Junín",
+      "name": "North Junín Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qvo",
+      "inverted_name": "Quechua, Napo Lowland",
+      "name": "Napo Lowland Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qvp",
+      "inverted_name": "Quechua, Pacaraos",
+      "name": "Pacaraos Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qvs",
+      "inverted_name": "Quechua, San Martín",
+      "name": "San Martín Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qvw",
+      "inverted_name": "Quechua, Huaylla Wanca",
+      "name": "Huaylla Wanca Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qvy",
+      "name": "Queyu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qvz",
+      "inverted_name": "Quichua, Northern Pastaza",
+      "name": "Northern Pastaza Quichua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qwa",
+      "inverted_name": "Quechua, Corongo Ancash",
+      "name": "Corongo Ancash Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qwc",
+      "inverted_name": "Quechua, Classical",
+      "name": "Classical Quechua",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "qwh",
+      "inverted_name": "Quechua, Huaylas Ancash",
+      "name": "Huaylas Ancash Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qwm",
+      "name": "Kuman (Russia)",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "qws",
+      "inverted_name": "Quechua, Sihuas Ancash",
+      "name": "Sihuas Ancash Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qwt",
+      "name": "Kwalhioqua-Tlatskanai",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "qxa",
+      "inverted_name": "Quechua, Chiquián Ancash",
+      "name": "Chiquián Ancash Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qxc",
+      "inverted_name": "Quechua, Chincha",
+      "name": "Chincha Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qxh",
+      "inverted_name": "Quechua, Panao Huánuco",
+      "name": "Panao Huánuco Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qxl",
+      "inverted_name": "Quichua, Salasaca Highland",
+      "name": "Salasaca Highland Quichua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qxn",
+      "inverted_name": "Quechua, Northern Conchucos Ancash",
+      "name": "Northern Conchucos Ancash Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qxo",
+      "inverted_name": "Quechua, Southern Conchucos Ancash",
+      "name": "Southern Conchucos Ancash Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qxp",
+      "inverted_name": "Quechua, Puno",
+      "name": "Puno Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qxq",
+      "name": "Qashqa'i",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qxr",
+      "inverted_name": "Quichua, Cañar Highland",
+      "name": "Cañar Highland Quichua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qxs",
+      "inverted_name": "Qiang, Southern",
+      "name": "Southern Qiang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qxt",
+      "inverted_name": "Quechua, Santa Ana de Tusi Pasco",
+      "name": "Santa Ana de Tusi Pasco Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qxu",
+      "inverted_name": "Quechua, Arequipa-La Unión",
+      "name": "Arequipa-La Unión Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qxw",
+      "inverted_name": "Quechua, Jauja Wanca",
+      "name": "Jauja Wanca Quechua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "qya",
+      "name": "Quenya",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "qyp",
+      "name": "Quiripi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "raa",
+      "name": "Dungmali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rab",
+      "name": "Camling",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rac",
+      "name": "Rasawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rad",
+      "name": "Rade",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "raf",
+      "inverted_name": "Meohang, Western",
+      "name": "Western Meohang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rag",
+      "name": "Logooli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rah",
+      "name": "Rabha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rai",
+      "name": "Ramoaaina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "raj",
+      "name": "Rajasthani",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rak",
+      "name": "Tulu-Bohuai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ral",
+      "name": "Ralte",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ram",
+      "name": "Canela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ran",
+      "name": "Riantana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rao",
+      "name": "Rao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rap",
+      "name": "Rapanui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "raq",
+      "name": "Saam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rar",
+      "name": "Rarotongan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ras",
+      "name": "Tegali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rat",
+      "name": "Razajerdi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rau",
+      "name": "Raute",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rav",
+      "name": "Sampang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "raw",
+      "name": "Rawang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rax",
+      "name": "Rang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ray",
+      "name": "Rapa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "raz",
+      "name": "Rahambuu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rbb",
+      "inverted_name": "Palaung, Rumai",
+      "name": "Rumai Palaung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rbk",
+      "inverted_name": "Bontok, Northern",
+      "name": "Northern Bontok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rbl",
+      "inverted_name": "Bikol, Miraya",
+      "name": "Miraya Bikol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rbp",
+      "name": "Barababaraba",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "rcf",
+      "inverted_name": "Creole French, Réunion",
+      "name": "Réunion Creole French",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rdb",
+      "name": "Rudbari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rea",
+      "name": "Rerau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "reb",
+      "name": "Rembong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ree",
+      "inverted_name": "Kayan, Rejang",
+      "name": "Rejang Kayan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "reg",
+      "name": "Kara (Tanzania)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rei",
+      "name": "Reli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rej",
+      "name": "Rejang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rel",
+      "name": "Rendille",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rem",
+      "name": "Remo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ren",
+      "name": "Rengao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rer",
+      "name": "Rer Bare",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "res",
+      "name": "Reshe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ret",
+      "name": "Retta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rey",
+      "name": "Reyesano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rga",
+      "name": "Roria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rge",
+      "name": "Romano-Greek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rgk",
+      "name": "Rangkas",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "rgn",
+      "name": "Romagnol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rgr",
+      "name": "Resígaro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rgs",
+      "inverted_name": "Roglai, Southern",
+      "name": "Southern Roglai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rgu",
+      "name": "Ringgou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rhg",
+      "name": "Rohingya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rhp",
+      "name": "Yahang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ria",
+      "name": "Riang (India)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rib",
+      "name": "Bribri Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rif",
+      "name": "Tarifit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ril",
+      "name": "Riang Lang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rim",
+      "name": "Nyaturu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rin",
+      "name": "Nungu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rir",
+      "name": "Ribun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rit",
+      "name": "Ritharrngu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "riu",
+      "name": "Riung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rjg",
+      "name": "Rajong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rji",
+      "name": "Raji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rjs",
+      "name": "Rajbanshi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rka",
+      "name": "Kraol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rkb",
+      "name": "Rikbaktsa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rkh",
+      "name": "Rakahanga-Manihiki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rki",
+      "name": "Rakhine",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rkm",
+      "name": "Marka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rkt",
+      "name": "Rangpuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rkw",
+      "name": "Arakwal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "rma",
+      "name": "Rama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmb",
+      "name": "Rembarrnga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmc",
+      "inverted_name": "Romani, Carpathian",
+      "name": "Carpathian Romani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmd",
+      "inverted_name": "Danish, Traveller",
+      "name": "Traveller Danish",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "rme",
+      "name": "Angloromani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmf",
+      "inverted_name": "Romani, Kalo Finnish",
+      "name": "Kalo Finnish Romani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmg",
+      "inverted_name": "Norwegian, Traveller",
+      "name": "Traveller Norwegian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmh",
+      "name": "Murkim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmi",
+      "name": "Lomavren",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmk",
+      "name": "Romkun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rml",
+      "inverted_name": "Romani, Baltic",
+      "name": "Baltic Romani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmm",
+      "name": "Roma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmn",
+      "inverted_name": "Romani, Balkan",
+      "name": "Balkan Romani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmo",
+      "inverted_name": "Romani, Sinte",
+      "name": "Sinte Romani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmp",
+      "name": "Rempi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmq",
+      "name": "Caló",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rms",
+      "name": "Romanian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmt",
+      "name": "Domari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmu",
+      "inverted_name": "Romani, Tavringer",
+      "name": "Tavringer Romani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmv",
+      "name": "Romanova",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "rmw",
+      "inverted_name": "Romani, Welsh",
+      "name": "Welsh Romani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmx",
+      "name": "Romam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmy",
+      "inverted_name": "Romani, Vlax",
+      "name": "Vlax Romani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rmz",
+      "name": "Marma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rnb",
+      "name": "Brunca Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rnd",
+      "name": "Ruund",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rng",
+      "name": "Ronga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rnl",
+      "name": "Ranglong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rnn",
+      "name": "Roon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rnp",
+      "name": "Rongpo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rnr",
+      "name": "Nari Nari",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "rnw",
+      "name": "Rungwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rob",
+      "name": "Tae'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "roc",
+      "inverted_name": "Roglai, Cacgia",
+      "name": "Cacgia Roglai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rod",
+      "name": "Rogo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "roe",
+      "name": "Ronji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rof",
+      "name": "Rombo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rog",
+      "inverted_name": "Roglai, Northern",
+      "name": "Northern Roglai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "rm",
+      "alpha_3": "roh",
+      "name": "Romansh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rol",
+      "name": "Romblomanon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rom",
+      "name": "Romany",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ro",
+      "alpha_3": "ron",
+      "bibliographic": "rum",
+      "name": "Romanian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "roo",
+      "name": "Rotokas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rop",
+      "name": "Kriol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ror",
+      "name": "Rongga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rou",
+      "name": "Runga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "row",
+      "name": "Dela-Oenale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rpn",
+      "name": "Repanbitip",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rpt",
+      "name": "Rapting",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rri",
+      "name": "Ririo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rro",
+      "name": "Waima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rrt",
+      "name": "Arritinngithigh",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "rsb",
+      "name": "Romano-Serbian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rsk",
+      "name": "Ruthenian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rsl",
+      "name": "Russian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rsm",
+      "name": "Miriwoong Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rsn",
+      "name": "Rwandan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rtc",
+      "inverted_name": "Chin, Rungtu",
+      "name": "Rungtu Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rth",
+      "name": "Ratahan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rtm",
+      "name": "Rotuman",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rts",
+      "name": "Yurats",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "rtw",
+      "name": "Rathawi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rub",
+      "name": "Gungu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ruc",
+      "name": "Ruuli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rue",
+      "name": "Rusyn",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ruf",
+      "name": "Luguru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rug",
+      "name": "Roviana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ruh",
+      "name": "Ruga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rui",
+      "name": "Rufiji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ruk",
+      "name": "Che",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "rn",
+      "alpha_3": "run",
+      "name": "Rundi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ruo",
+      "inverted_name": "Romanian, Istro",
+      "name": "Istro Romanian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rup",
+      "inverted_name": "Romanian, Macedo-",
+      "name": "Macedo-Romanian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ruq",
+      "inverted_name": "Romanian, Megleno",
+      "name": "Megleno Romanian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ru",
+      "alpha_3": "rus",
+      "name": "Russian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rut",
+      "name": "Rutul",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ruu",
+      "inverted_name": "Lobu, Lanas",
+      "name": "Lanas Lobu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ruy",
+      "name": "Mala (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ruz",
+      "name": "Ruma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rwa",
+      "name": "Rawo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rwk",
+      "name": "Rwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rwl",
+      "name": "Ruwila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rwm",
+      "name": "Amba (Uganda)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rwo",
+      "name": "Rawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rwr",
+      "name": "Marwari (India)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rxd",
+      "name": "Ngardi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rxw",
+      "name": "Karuwali",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ryn",
+      "inverted_name": "Amami-Oshima, Northern",
+      "name": "Northern Amami-Oshima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rys",
+      "name": "Yaeyama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ryu",
+      "inverted_name": "Okinawan, Central",
+      "name": "Central Okinawan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "rzh",
+      "name": "Rāziḥī",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "saa",
+      "name": "Saba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sab",
+      "name": "Buglere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sac",
+      "name": "Meskwaki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sad",
+      "name": "Sandawe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sae",
+      "name": "Sabanê",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "saf",
+      "name": "Safaliba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "sg",
+      "alpha_3": "sag",
+      "name": "Sango",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sah",
+      "name": "Yakut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "saj",
+      "name": "Sahu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sak",
+      "name": "Sake",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sam",
+      "inverted_name": "Aramaic, Samaritan",
+      "name": "Samaritan Aramaic",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_2": "sa",
+      "alpha_3": "san",
+      "name": "Sanskrit",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "sao",
+      "name": "Sause",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "saq",
+      "name": "Samburu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sar",
+      "name": "Saraveca",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "sas",
+      "name": "Sasak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sat",
+      "name": "Santali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sau",
+      "name": "Saleman",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sav",
+      "name": "Saafi-Saafi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "saw",
+      "name": "Sawi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sax",
+      "name": "Sa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "say",
+      "name": "Saya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "saz",
+      "name": "Saurashtra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sba",
+      "name": "Ngambay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbb",
+      "name": "Simbo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbc",
+      "name": "Kele (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbd",
+      "inverted_name": "Samo, Southern",
+      "name": "Southern Samo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbe",
+      "name": "Saliba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbf",
+      "name": "Chabu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbg",
+      "name": "Seget",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbh",
+      "name": "Sori-Harengan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbi",
+      "name": "Seti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbj",
+      "name": "Surbakhal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbk",
+      "name": "Safwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbl",
+      "inverted_name": "Sambal, Botolan",
+      "name": "Botolan Sambal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbm",
+      "name": "Sagala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbn",
+      "inverted_name": "Bhil, Sindhi",
+      "name": "Sindhi Bhil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbo",
+      "name": "Sabüm",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbp",
+      "name": "Sangu (Tanzania)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbq",
+      "name": "Sileibi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbr",
+      "name": "Sembakung Murut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbs",
+      "name": "Subiya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbt",
+      "name": "Kimki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbu",
+      "inverted_name": "Bhoti, Stod",
+      "name": "Stod Bhoti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbv",
+      "name": "Sabine",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "sbw",
+      "name": "Simba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbx",
+      "name": "Seberuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sby",
+      "name": "Soli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sbz",
+      "name": "Sara Kaba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "scb",
+      "name": "Chut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sce",
+      "name": "Dongxiang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "scf",
+      "inverted_name": "Creole French, San Miguel",
+      "name": "San Miguel Creole French",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "scg",
+      "name": "Sanggau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sch",
+      "name": "Sakachep",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sci",
+      "inverted_name": "Creole Malay, Sri Lankan",
+      "name": "Sri Lankan Creole Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sck",
+      "name": "Sadri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "scl",
+      "name": "Shina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "scn",
+      "name": "Sicilian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sco",
+      "name": "Scots",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "scp",
+      "name": "Hyolmo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "scq",
+      "name": "Sa'och",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "scs",
+      "inverted_name": "Slavey, North",
+      "name": "North Slavey",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sct",
+      "inverted_name": "Katang, Southern",
+      "name": "Southern Katang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "scu",
+      "name": "Shumcho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "scv",
+      "name": "Sheni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "scw",
+      "name": "Sha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "scx",
+      "name": "Sicel",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "sda",
+      "name": "Toraja-Sa'dan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdb",
+      "name": "Shabak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdc",
+      "inverted_name": "Sardinian, Sassarese",
+      "name": "Sassarese Sardinian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sde",
+      "name": "Surubu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdf",
+      "name": "Sarli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdg",
+      "name": "Savi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdh",
+      "inverted_name": "Kurdish, Southern",
+      "name": "Southern Kurdish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdj",
+      "name": "Suundi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdk",
+      "name": "Sos Kundi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdl",
+      "name": "Saudi Arabian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdn",
+      "inverted_name": "Sardinian, Gallurese",
+      "name": "Gallurese Sardinian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdo",
+      "inverted_name": "Bidayuh, Bukar-Sadung",
+      "name": "Bukar-Sadung Bidayuh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdp",
+      "name": "Sherdukpen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdq",
+      "name": "Semandang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdr",
+      "inverted_name": "Sadri, Oraon",
+      "name": "Oraon Sadri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sds",
+      "name": "Sened",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "sdt",
+      "name": "Shuadit",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "sdu",
+      "name": "Sarudu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdx",
+      "inverted_name": "Melanau, Sibu",
+      "name": "Sibu Melanau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sdz",
+      "name": "Sallands",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sea",
+      "name": "Semai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "seb",
+      "inverted_name": "Senoufo, Shempire",
+      "name": "Shempire Senoufo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sec",
+      "name": "Sechelt",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sed",
+      "name": "Sedang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "see",
+      "name": "Seneca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sef",
+      "inverted_name": "Senoufo, Cebaara",
+      "name": "Cebaara Senoufo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "seg",
+      "name": "Segeju",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "seh",
+      "name": "Sena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sei",
+      "name": "Seri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sej",
+      "name": "Sene",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sek",
+      "name": "Sekani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sel",
+      "name": "Selkup",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sen",
+      "inverted_name": "Sénoufo, Nanerigé",
+      "name": "Nanerigé Sénoufo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "seo",
+      "name": "Suarmin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sep",
+      "inverted_name": "Sénoufo, Sìcìté",
+      "name": "Sìcìté Sénoufo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "seq",
+      "inverted_name": "Sénoufo, Senara",
+      "name": "Senara Sénoufo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ser",
+      "name": "Serrano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ses",
+      "inverted_name": "Songhai, Koyraboro Senni",
+      "name": "Koyraboro Senni Songhai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "set",
+      "name": "Sentani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "seu",
+      "name": "Serui-Laut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sev",
+      "inverted_name": "Senoufo, Nyarafolo",
+      "name": "Nyarafolo Senoufo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sew",
+      "name": "Sewa Bay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sey",
+      "name": "Secoya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sez",
+      "inverted_name": "Chin, Senthang",
+      "name": "Senthang Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sfb",
+      "name": "Langue des signes de Belgique Francophone",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sfe",
+      "inverted_name": "Subanen, Eastern",
+      "name": "Eastern Subanen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sfm",
+      "inverted_name": "Miao, Small Flowery",
+      "name": "Small Flowery Miao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sfs",
+      "name": "South African Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sfw",
+      "name": "Sehwi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sga",
+      "inverted_name": "Irish, Old (to 900)",
+      "name": "Old Irish (to 900)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "sgb",
+      "inverted_name": "Ayta, Mag-antsi",
+      "name": "Mag-antsi Ayta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgc",
+      "name": "Kipsigis",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgd",
+      "name": "Surigaonon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sge",
+      "name": "Segai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgg",
+      "name": "Swiss-German Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgh",
+      "name": "Shughni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgi",
+      "name": "Suga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgj",
+      "name": "Surgujia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgk",
+      "name": "Sangkong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgm",
+      "name": "Singa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "sgp",
+      "name": "Singpho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgr",
+      "name": "Sangisari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgs",
+      "name": "Samogitian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgt",
+      "name": "Brokpake",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgu",
+      "name": "Salas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgw",
+      "name": "Sebat Bet Gurage",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgx",
+      "name": "Sierra Leone Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgy",
+      "name": "Sanglechi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sgz",
+      "name": "Sursurunga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sha",
+      "name": "Shall-Zwall",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shb",
+      "name": "Ninam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shc",
+      "name": "Sonde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shd",
+      "name": "Kundal Shahi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "she",
+      "name": "Sheko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shg",
+      "name": "Shua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shh",
+      "name": "Shoshoni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shi",
+      "name": "Tachelhit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shj",
+      "name": "Shatt",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shk",
+      "name": "Shilluk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shl",
+      "name": "Shendu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shm",
+      "name": "Shahrudi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shn",
+      "name": "Shan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sho",
+      "name": "Shanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shp",
+      "name": "Shipibo-Conibo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shq",
+      "name": "Sala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shr",
+      "name": "Shi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shs",
+      "name": "Shuswap",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sht",
+      "name": "Shasta",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "shu",
+      "inverted_name": "Arabic, Chadian",
+      "name": "Chadian Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shv",
+      "name": "Shehri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shw",
+      "name": "Shwai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shx",
+      "name": "She",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shy",
+      "name": "Tachawit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "shz",
+      "inverted_name": "Senoufo, Syenara",
+      "name": "Syenara Senoufo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sia",
+      "inverted_name": "Sami, Akkala",
+      "name": "Akkala Sami",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "sib",
+      "name": "Sebop",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sid",
+      "name": "Sidamo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sie",
+      "name": "Simaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sif",
+      "name": "Siamou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sig",
+      "name": "Paasaal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sih",
+      "name": "Zire",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sii",
+      "name": "Shom Peng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sij",
+      "name": "Numbami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sik",
+      "name": "Sikiana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sil",
+      "inverted_name": "Sisaala, Tumulung",
+      "name": "Tumulung Sisaala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sim",
+      "name": "Mende (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "si",
+      "alpha_3": "sin",
+      "name": "Sinhala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sip",
+      "name": "Sikkimese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "siq",
+      "name": "Sonia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sir",
+      "name": "Siri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sis",
+      "name": "Siuslaw",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "siu",
+      "name": "Sinagen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "siv",
+      "name": "Sumariup",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "siw",
+      "name": "Siwai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "six",
+      "name": "Sumau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "siy",
+      "name": "Sivandi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "siz",
+      "name": "Siwi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sja",
+      "name": "Epena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sjb",
+      "name": "Sajau Basap",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sjd",
+      "inverted_name": "Sami, Kildin",
+      "name": "Kildin Sami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sje",
+      "inverted_name": "Sami, Pite",
+      "name": "Pite Sami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sjg",
+      "name": "Assangori",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sjk",
+      "inverted_name": "Sami, Kemi",
+      "name": "Kemi Sami",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "sjl",
+      "name": "Sajalong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sjm",
+      "name": "Mapun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sjn",
+      "name": "Sindarin",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "sjo",
+      "name": "Xibe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sjp",
+      "name": "Surjapuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sjr",
+      "name": "Siar-Lak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sjs",
+      "name": "Senhaja De Srair",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "sjt",
+      "inverted_name": "Sami, Ter",
+      "name": "Ter Sami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sju",
+      "inverted_name": "Sami, Ume",
+      "name": "Ume Sami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sjw",
+      "name": "Shawnee",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ska",
+      "name": "Skagit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skb",
+      "name": "Saek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skc",
+      "name": "Ma Manda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skd",
+      "inverted_name": "Miwok, Southern Sierra",
+      "name": "Southern Sierra Miwok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ske",
+      "name": "Seke (Vanuatu)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skf",
+      "name": "Sakirabiá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skg",
+      "inverted_name": "Malagasy, Sakalava",
+      "name": "Sakalava Malagasy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skh",
+      "name": "Sikule",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ski",
+      "name": "Sika",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skj",
+      "name": "Seke (Nepal)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skm",
+      "name": "Kutong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skn",
+      "inverted_name": "Subanon, Kolibugan",
+      "name": "Kolibugan Subanon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sko",
+      "name": "Seko Tengah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skp",
+      "name": "Sekapan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skq",
+      "name": "Sininkere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skr",
+      "name": "Saraiki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sks",
+      "name": "Maia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skt",
+      "name": "Sakata",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sku",
+      "name": "Sakao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skv",
+      "name": "Skou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skw",
+      "inverted_name": "Creole Dutch, Skepi",
+      "name": "Skepi Creole Dutch",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "skx",
+      "name": "Seko Padang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sky",
+      "name": "Sikaiana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "skz",
+      "name": "Sekar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "slc",
+      "name": "Sáliba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sld",
+      "name": "Sissala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sle",
+      "name": "Sholaga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "slf",
+      "name": "Swiss-Italian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "slg",
+      "name": "Selungai Murut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "slh",
+      "inverted_name": "Salish, Southern Puget Sound",
+      "name": "Southern Puget Sound Salish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sli",
+      "inverted_name": "Silesian, Lower",
+      "name": "Lower Silesian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "slj",
+      "name": "Salumá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "sk",
+      "alpha_3": "slk",
+      "bibliographic": "slo",
+      "name": "Slovak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sll",
+      "name": "Salt-Yui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "slm",
+      "inverted_name": "Sama, Pangutaran",
+      "name": "Pangutaran Sama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sln",
+      "name": "Salinan",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "slp",
+      "name": "Lamaholot",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "slq",
+      "name": "Salchuq",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "slr",
+      "name": "Salar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sls",
+      "name": "Singapore Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "slt",
+      "name": "Sila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "slu",
+      "name": "Selaru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "sl",
+      "alpha_3": "slv",
+      "name": "Slovenian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "slw",
+      "name": "Sialum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "slx",
+      "name": "Salampasu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sly",
+      "name": "Selayar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "slz",
+      "name": "Ma'ya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sma",
+      "inverted_name": "Sami, Southern",
+      "name": "Southern Sami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smb",
+      "name": "Simbari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smc",
+      "name": "Som",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_2": "se",
+      "alpha_3": "sme",
+      "inverted_name": "Sami, Northern",
+      "name": "Northern Sami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smf",
+      "name": "Auwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smg",
+      "name": "Simbali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smh",
+      "name": "Samei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smj",
+      "name": "Lule Sami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smk",
+      "name": "Bolinao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sml",
+      "inverted_name": "Sama, Central",
+      "name": "Central Sama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smm",
+      "name": "Musasa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smn",
+      "inverted_name": "Sami, Inari",
+      "name": "Inari Sami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "sm",
+      "alpha_3": "smo",
+      "name": "Samoan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smp",
+      "name": "Samaritan",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "smq",
+      "name": "Samo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smr",
+      "name": "Simeulue",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sms",
+      "inverted_name": "Sami, Skolt",
+      "name": "Skolt Sami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smt",
+      "name": "Simte",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smu",
+      "name": "Somray",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "smv",
+      "name": "Samvedi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smw",
+      "name": "Sumbawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smx",
+      "name": "Samba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smy",
+      "name": "Semnani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "smz",
+      "name": "Simeku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "sn",
+      "alpha_3": "sna",
+      "name": "Shona",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snc",
+      "name": "Sinaugoro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "sd",
+      "alpha_3": "snd",
+      "name": "Sindhi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sne",
+      "inverted_name": "Bidayuh, Bau",
+      "name": "Bau Bidayuh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snf",
+      "name": "Noon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sng",
+      "name": "Sanga (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sni",
+      "name": "Sensi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "snj",
+      "inverted_name": "Sango, Riverain",
+      "name": "Riverain Sango",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snk",
+      "name": "Soninke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snl",
+      "name": "Sangil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snm",
+      "inverted_name": "Ma'di, Southern",
+      "name": "Southern Ma'di",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snn",
+      "name": "Siona",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sno",
+      "name": "Snohomish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snp",
+      "name": "Siane",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snq",
+      "name": "Sangu (Gabon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snr",
+      "name": "Sihan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sns",
+      "name": "South West Bay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snu",
+      "name": "Senggi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snv",
+      "name": "Sa'ban",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snw",
+      "name": "Selee",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snx",
+      "name": "Sam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sny",
+      "name": "Saniyo-Hiyewe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "snz",
+      "name": "Kou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "soa",
+      "name": "Thai Song",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sob",
+      "name": "Sobei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "soc",
+      "name": "So (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sod",
+      "name": "Songoora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "soe",
+      "name": "Songomeno",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sog",
+      "name": "Sogdian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "soh",
+      "name": "Aka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "soi",
+      "name": "Sonha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "soj",
+      "name": "Soi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sok",
+      "name": "Sokoro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sol",
+      "name": "Solos",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "so",
+      "alpha_3": "som",
+      "name": "Somali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "soo",
+      "name": "Songo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sop",
+      "name": "Songe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "soq",
+      "name": "Kanasi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sor",
+      "name": "Somrai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sos",
+      "name": "Seeku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "st",
+      "alpha_3": "sot",
+      "inverted_name": "Sotho, Southern",
+      "name": "Southern Sotho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sou",
+      "inverted_name": "Thai, Southern",
+      "name": "Southern Thai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sov",
+      "name": "Sonsorol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sow",
+      "name": "Sowanda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sox",
+      "name": "Swo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "soy",
+      "name": "Miyobe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "soz",
+      "name": "Temi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "es",
+      "alpha_3": "spa",
+      "name": "Spanish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spb",
+      "name": "Sepa (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spc",
+      "name": "Sapé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spd",
+      "name": "Saep",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spe",
+      "name": "Sepa (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spg",
+      "name": "Sian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spi",
+      "name": "Saponi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spk",
+      "name": "Sengo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spl",
+      "name": "Selepet",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spm",
+      "name": "Akukem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spn",
+      "name": "Sanapaná",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spo",
+      "name": "Spokane",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spp",
+      "inverted_name": "Senoufo, Supyire",
+      "name": "Supyire Senoufo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spq",
+      "inverted_name": "Spanish, Loreto-Ucayali",
+      "name": "Loreto-Ucayali Spanish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spr",
+      "name": "Saparua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sps",
+      "name": "Saposa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spt",
+      "inverted_name": "Bhoti, Spiti",
+      "name": "Spiti Bhoti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spu",
+      "name": "Sapuan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spv",
+      "name": "Sambalpuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "spx",
+      "inverted_name": "Picene, South",
+      "name": "South Picene",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "spy",
+      "name": "Sabaot",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sqa",
+      "name": "Shama-Sambuga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sqh",
+      "name": "Shau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "sq",
+      "alpha_3": "sqi",
+      "bibliographic": "alb",
+      "name": "Albanian",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sqk",
+      "name": "Albanian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sqm",
+      "name": "Suma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sqn",
+      "name": "Susquehannock",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "sqo",
+      "name": "Sorkhei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sqq",
+      "name": "Sou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sqr",
+      "inverted_name": "Arabic, Siculo",
+      "name": "Siculo Arabic",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "sqs",
+      "name": "Sri Lankan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sqt",
+      "name": "Soqotri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "squ",
+      "name": "Squamish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sqx",
+      "name": "Kufr Qassem Sign Language (KQSL)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sra",
+      "name": "Saruga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srb",
+      "name": "Sora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "src",
+      "inverted_name": "Sardinian, Logudorese",
+      "name": "Logudorese Sardinian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "sc",
+      "alpha_3": "srd",
+      "name": "Sardinian",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sre",
+      "name": "Sara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srf",
+      "name": "Nafi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srg",
+      "name": "Sulod",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srh",
+      "name": "Sarikoli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sri",
+      "name": "Siriano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srk",
+      "name": "Serudung Murut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srl",
+      "name": "Isirawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srm",
+      "name": "Saramaccan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srn",
+      "name": "Sranan Tongo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sro",
+      "inverted_name": "Sardinian, Campidanese",
+      "name": "Campidanese Sardinian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "sr",
+      "alpha_3": "srp",
+      "name": "Serbian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srq",
+      "name": "Sirionó",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srr",
+      "name": "Serer",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srs",
+      "name": "Sarsi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srt",
+      "name": "Sauri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sru",
+      "name": "Suruí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srv",
+      "inverted_name": "Sorsoganon, Southern",
+      "name": "Southern Sorsoganon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srw",
+      "name": "Serua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srx",
+      "name": "Sirmauri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sry",
+      "name": "Sera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "srz",
+      "name": "Shahmirzadi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssb",
+      "inverted_name": "Sama, Southern",
+      "name": "Southern Sama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssc",
+      "name": "Suba-Simbiti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssd",
+      "name": "Siroi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sse",
+      "name": "Balangingi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssf",
+      "name": "Thao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssg",
+      "name": "Seimat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssh",
+      "inverted_name": "Arabic, Shihhi",
+      "name": "Shihhi Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssi",
+      "name": "Sansi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssj",
+      "name": "Sausi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssk",
+      "name": "Sunam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssl",
+      "inverted_name": "Sisaala, Western",
+      "name": "Western Sisaala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssm",
+      "name": "Semnam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssn",
+      "name": "Waata",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sso",
+      "name": "Sissano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssp",
+      "name": "Spanish Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssq",
+      "name": "So'a",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssr",
+      "name": "Swiss-French Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sss",
+      "name": "Sô",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sst",
+      "name": "Sinasina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssu",
+      "name": "Susuami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssv",
+      "name": "Shark Bay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ss",
+      "alpha_3": "ssw",
+      "name": "Swati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssx",
+      "name": "Samberigi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssy",
+      "name": "Saho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ssz",
+      "name": "Sengseng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sta",
+      "name": "Settla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stb",
+      "inverted_name": "Subanen, Northern",
+      "name": "Northern Subanen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "std",
+      "name": "Sentinel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ste",
+      "name": "Liana-Seti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stf",
+      "name": "Seta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stg",
+      "name": "Trieng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sth",
+      "name": "Shelta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sti",
+      "inverted_name": "Stieng, Bulo",
+      "name": "Bulo Stieng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stj",
+      "inverted_name": "Samo, Matya",
+      "name": "Matya Samo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stk",
+      "name": "Arammba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stl",
+      "name": "Stellingwerfs",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stm",
+      "name": "Setaman",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stn",
+      "name": "Owa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sto",
+      "name": "Stoney",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stp",
+      "inverted_name": "Tepehuan, Southeastern",
+      "name": "Southeastern Tepehuan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stq",
+      "name": "Saterfriesisch",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "str",
+      "inverted_name": "Salish, Straits",
+      "name": "Straits Salish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sts",
+      "name": "Shumashti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stt",
+      "inverted_name": "Stieng, Budeh",
+      "name": "Budeh Stieng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stu",
+      "name": "Samtao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stv",
+      "name": "Silt'e",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "stw",
+      "name": "Satawalese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sty",
+      "inverted_name": "Tatar, Siberian",
+      "name": "Siberian Tatar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sua",
+      "name": "Sulka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sub",
+      "name": "Suku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "suc",
+      "inverted_name": "Subanon, Western",
+      "name": "Western Subanon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sue",
+      "name": "Suena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sug",
+      "name": "Suganga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sui",
+      "name": "Suki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "suj",
+      "name": "Shubi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "suk",
+      "name": "Sukuma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "su",
+      "alpha_3": "sun",
+      "name": "Sundanese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "suo",
+      "name": "Bouni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "suq",
+      "inverted_name": "Suri, Tirmaga-Chai",
+      "name": "Tirmaga-Chai Suri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sur",
+      "name": "Mwaghavul",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sus",
+      "name": "Susu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sut",
+      "name": "Subtiaba",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "suv",
+      "name": "Puroik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "suw",
+      "name": "Sumbwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sux",
+      "name": "Sumerian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "suy",
+      "name": "Suyá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "suz",
+      "name": "Sunwar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sva",
+      "name": "Svan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "svb",
+      "name": "Ulau-Suain",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "svc",
+      "inverted_name": "Creole English, Vincentian",
+      "name": "Vincentian Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sve",
+      "name": "Serili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "svk",
+      "name": "Slovakian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "svm",
+      "name": "Slavomolisano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "svs",
+      "name": "Savosavo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "svx",
+      "name": "Skalvian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_2": "sw",
+      "alpha_3": "swa",
+      "name": "Swahili (macrolanguage)",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swb",
+      "inverted_name": "Comorian, Maore",
+      "name": "Maore Comorian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swc",
+      "inverted_name": "Swahili, Congo",
+      "name": "Congo Swahili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "sv",
+      "alpha_3": "swe",
+      "name": "Swedish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swf",
+      "name": "Sere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swg",
+      "name": "Swabian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swh",
+      "name": "Swahili (individual language)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swi",
+      "name": "Sui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swj",
+      "name": "Sira",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swk",
+      "inverted_name": "Sena, Malawi",
+      "name": "Malawi Sena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swl",
+      "name": "Swedish Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swm",
+      "name": "Samosa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swn",
+      "name": "Sawknah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swo",
+      "name": "Shanenawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swp",
+      "name": "Suau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swq",
+      "name": "Sharwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swr",
+      "name": "Saweru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sws",
+      "name": "Seluwasan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swt",
+      "name": "Sawila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swu",
+      "name": "Suwawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swv",
+      "name": "Shekhawati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sww",
+      "name": "Sowa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "swx",
+      "name": "Suruahá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "swy",
+      "name": "Sarua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sxb",
+      "name": "Suba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sxc",
+      "name": "Sicanian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "sxe",
+      "name": "Sighu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sxg",
+      "name": "Shuhi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sxk",
+      "inverted_name": "Kalapuya, Southern",
+      "name": "Southern Kalapuya",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "sxl",
+      "name": "Selian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "sxm",
+      "name": "Samre",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sxn",
+      "name": "Sangir",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sxo",
+      "name": "Sorothaptic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "sxr",
+      "name": "Saaroa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sxs",
+      "name": "Sasaru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sxu",
+      "inverted_name": "Saxon, Upper",
+      "name": "Upper Saxon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sxw",
+      "inverted_name": "Gbe, Saxwe",
+      "name": "Saxwe Gbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sya",
+      "name": "Siang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "syb",
+      "inverted_name": "Subanen, Central",
+      "name": "Central Subanen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "syc",
+      "inverted_name": "Syriac, Classical",
+      "name": "Classical Syriac",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "syi",
+      "name": "Seki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "syk",
+      "name": "Sukur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "syl",
+      "name": "Sylheti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sym",
+      "inverted_name": "Samo, Maya",
+      "name": "Maya Samo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "syn",
+      "name": "Senaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "syo",
+      "name": "Suoy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "syr",
+      "name": "Syriac",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sys",
+      "name": "Sinyar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "syw",
+      "name": "Kagate",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "syx",
+      "name": "Samay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "syy",
+      "name": "Al-Sayyid Bedouin Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "sza",
+      "name": "Semelai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "szb",
+      "name": "Ngalum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "szc",
+      "name": "Semaq Beri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "szd",
+      "name": "Seru",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "sze",
+      "name": "Seze",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "szg",
+      "name": "Sengele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "szl",
+      "name": "Silesian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "szn",
+      "name": "Sula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "szp",
+      "name": "Suabo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "szs",
+      "name": "Solomon Islands Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "szv",
+      "name": "Isu (Fako Division)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "szw",
+      "name": "Sawai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "szy",
+      "name": "Sakizaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "taa",
+      "inverted_name": "Tanana, Lower",
+      "name": "Lower Tanana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tab",
+      "name": "Tabassaran",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tac",
+      "inverted_name": "Tarahumara, Lowland",
+      "name": "Lowland Tarahumara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tad",
+      "name": "Tause",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tae",
+      "name": "Tariana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "taf",
+      "name": "Tapirapé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tag",
+      "name": "Tagoi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ty",
+      "alpha_3": "tah",
+      "name": "Tahitian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "taj",
+      "inverted_name": "Tamang, Eastern",
+      "name": "Eastern Tamang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tak",
+      "name": "Tala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tal",
+      "name": "Tal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ta",
+      "alpha_3": "tam",
+      "name": "Tamil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tan",
+      "name": "Tangale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tao",
+      "name": "Yami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tap",
+      "name": "Taabwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "taq",
+      "name": "Tamasheq",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tar",
+      "inverted_name": "Tarahumara, Central",
+      "name": "Central Tarahumara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tas",
+      "name": "Tay Boi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_2": "tt",
+      "alpha_3": "tat",
+      "name": "Tatar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tau",
+      "inverted_name": "Tanana, Upper",
+      "name": "Upper Tanana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tav",
+      "name": "Tatuyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "taw",
+      "name": "Tai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tax",
+      "name": "Tamki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tay",
+      "name": "Atayal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "taz",
+      "name": "Tocho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tba",
+      "name": "Aikanã",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbc",
+      "name": "Takia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbd",
+      "name": "Kaki Ae",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbe",
+      "name": "Tanimbili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbf",
+      "name": "Mandara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbg",
+      "inverted_name": "Tairora, North",
+      "name": "North Tairora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbh",
+      "name": "Dharawal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tbi",
+      "name": "Gaam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbj",
+      "name": "Tiang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbk",
+      "inverted_name": "Tagbanwa, Calamian",
+      "name": "Calamian Tagbanwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbl",
+      "name": "Tboli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbm",
+      "name": "Tagbu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbn",
+      "inverted_name": "Tunebo, Barro Negro",
+      "name": "Barro Negro Tunebo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbo",
+      "name": "Tawala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbp",
+      "name": "Taworta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbr",
+      "name": "Tumtum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbs",
+      "name": "Tanguat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbt",
+      "name": "Tembo (Kitembo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbu",
+      "name": "Tubar",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tbv",
+      "name": "Tobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbw",
+      "name": "Tagbanwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbx",
+      "name": "Kapin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tby",
+      "name": "Tabaru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tbz",
+      "name": "Ditammari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tca",
+      "name": "Ticuna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcb",
+      "name": "Tanacross",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcc",
+      "name": "Datooga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcd",
+      "name": "Tafi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tce",
+      "inverted_name": "Tutchone, Southern",
+      "name": "Southern Tutchone",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcf",
+      "inverted_name": "Me'phaa, Malinaltepec",
+      "name": "Malinaltepec Me'phaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcg",
+      "name": "Tamagario",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tch",
+      "inverted_name": "Creole English, Turks And Caicos",
+      "name": "Turks And Caicos Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tci",
+      "name": "Wára",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tck",
+      "name": "Tchitchege",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcl",
+      "name": "Taman (Myanmar)",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tcm",
+      "name": "Tanahmerah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcn",
+      "name": "Tichurong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tco",
+      "name": "Taungyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcp",
+      "inverted_name": "Chin, Tawr",
+      "name": "Tawr Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcq",
+      "name": "Kaiy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcs",
+      "inverted_name": "Creole, Torres Strait",
+      "name": "Torres Strait Creole",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tct",
+      "name": "T'en",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcu",
+      "inverted_name": "Tarahumara, Southeastern",
+      "name": "Southeastern Tarahumara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcw",
+      "inverted_name": "Totonac, Tecpatlán",
+      "name": "Tecpatlán Totonac",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcx",
+      "name": "Toda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcy",
+      "name": "Tulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tcz",
+      "inverted_name": "Chin, Thado",
+      "name": "Thado Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tda",
+      "name": "Tagdal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdb",
+      "name": "Panchpargania",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdc",
+      "name": "Emberá-Tadó",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdd",
+      "name": "Tai Nüa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tde",
+      "inverted_name": "Dogon, Tiranige Diga",
+      "name": "Tiranige Diga Dogon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdf",
+      "name": "Talieng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdg",
+      "inverted_name": "Tamang, Western",
+      "name": "Western Tamang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdh",
+      "name": "Thulung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdi",
+      "name": "Tomadino",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdj",
+      "name": "Tajio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdk",
+      "name": "Tambas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdl",
+      "name": "Sur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdm",
+      "name": "Taruma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdn",
+      "name": "Tondano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdo",
+      "name": "Teme",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdq",
+      "name": "Tita",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdr",
+      "name": "Todrah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tds",
+      "name": "Doutai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdt",
+      "name": "Tetun Dili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdv",
+      "name": "Toro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdx",
+      "inverted_name": "Malagasy, Tandroy-Mahafaly",
+      "name": "Tandroy-Mahafaly Malagasy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tdy",
+      "name": "Tadyawan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tea",
+      "name": "Temiar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "teb",
+      "name": "Tetete",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tec",
+      "name": "Terik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ted",
+      "inverted_name": "Krumen, Tepo",
+      "name": "Tepo Krumen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tee",
+      "inverted_name": "Tepehua, Huehuetla",
+      "name": "Huehuetla Tepehua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tef",
+      "name": "Teressa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "teg",
+      "name": "Teke-Tege",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "teh",
+      "name": "Tehuelche",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tei",
+      "name": "Torricelli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tek",
+      "inverted_name": "Teke, Ibali",
+      "name": "Ibali Teke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "te",
+      "alpha_3": "tel",
+      "name": "Telugu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tem",
+      "name": "Timne",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ten",
+      "name": "Tama (Colombia)",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "teo",
+      "name": "Teso",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tep",
+      "name": "Tepecano",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "teq",
+      "name": "Temein",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ter",
+      "name": "Tereno",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tes",
+      "name": "Tengger",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tet",
+      "name": "Tetum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "teu",
+      "name": "Soo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tev",
+      "name": "Teor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tew",
+      "name": "Tewa (USA)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tex",
+      "name": "Tennet",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tey",
+      "name": "Tulishi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tez",
+      "name": "Tetserret",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tfi",
+      "inverted_name": "Gbe, Tofin",
+      "name": "Tofin Gbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tfn",
+      "name": "Tanaina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tfo",
+      "name": "Tefaro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tfr",
+      "name": "Teribe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tft",
+      "name": "Ternate",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tga",
+      "name": "Sagalla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgb",
+      "name": "Tobilung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgc",
+      "name": "Tigak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgd",
+      "name": "Ciwogai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tge",
+      "inverted_name": "Tamang, Eastern Gorkha",
+      "name": "Eastern Gorkha Tamang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgf",
+      "name": "Chalikha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgh",
+      "inverted_name": "Creole English, Tobagonian",
+      "name": "Tobagonian Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgi",
+      "name": "Lawunuia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgj",
+      "name": "Tagin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "tg",
+      "alpha_3": "tgk",
+      "name": "Tajik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "tl",
+      "alpha_3": "tgl",
+      "name": "Tagalog",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgn",
+      "name": "Tandaganon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgo",
+      "name": "Sudest",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgp",
+      "name": "Tangoa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgq",
+      "name": "Tring",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgr",
+      "name": "Tareng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgs",
+      "name": "Nume",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgt",
+      "inverted_name": "Tagbanwa, Central",
+      "name": "Central Tagbanwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgu",
+      "name": "Tanggu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgv",
+      "name": "Tingui-Boto",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tgw",
+      "inverted_name": "Senoufo, Tagwana",
+      "name": "Tagwana Senoufo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgx",
+      "name": "Tagish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tgy",
+      "name": "Togoyo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tgz",
+      "name": "Tagalaka",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_2": "th",
+      "alpha_3": "tha",
+      "name": "Thai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thd",
+      "name": "Kuuk Thaayorre",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "the",
+      "inverted_name": "Tharu, Chitwania",
+      "name": "Chitwania Tharu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thf",
+      "name": "Thangmi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thh",
+      "inverted_name": "Tarahumara, Northern",
+      "name": "Northern Tarahumara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thi",
+      "name": "Tai Long",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thk",
+      "name": "Tharaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thl",
+      "inverted_name": "Tharu, Dangaura",
+      "name": "Dangaura Tharu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thm",
+      "name": "Aheu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thn",
+      "name": "Thachanadan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thp",
+      "name": "Thompson",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thq",
+      "inverted_name": "Tharu, Kochila",
+      "name": "Kochila Tharu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thr",
+      "inverted_name": "Tharu, Rana",
+      "name": "Rana Tharu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ths",
+      "name": "Thakali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tht",
+      "name": "Tahltan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thu",
+      "name": "Thuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thv",
+      "inverted_name": "Tamahaq, Tahaggart",
+      "name": "Tahaggart Tamahaq",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thy",
+      "name": "Tha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "thz",
+      "inverted_name": "Tamajeq, Tayart",
+      "name": "Tayart Tamajeq",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tia",
+      "inverted_name": "Tamazight, Tidikelt",
+      "name": "Tidikelt Tamazight",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tic",
+      "name": "Tira",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tif",
+      "name": "Tifal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tig",
+      "name": "Tigre",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tih",
+      "inverted_name": "Murut, Timugon",
+      "name": "Timugon Murut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tii",
+      "name": "Tiene",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tij",
+      "name": "Tilung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tik",
+      "name": "Tikar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "til",
+      "name": "Tillamook",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tim",
+      "name": "Timbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tin",
+      "name": "Tindi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tio",
+      "name": "Teop",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tip",
+      "name": "Trimuris",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tiq",
+      "name": "Tiéfo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ti",
+      "alpha_3": "tir",
+      "name": "Tigrinya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tis",
+      "inverted_name": "Itneg, Masadiit",
+      "name": "Masadiit Itneg",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tit",
+      "name": "Tinigua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tiu",
+      "name": "Adasen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tiv",
+      "name": "Tiv",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tiw",
+      "name": "Tiwi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tix",
+      "inverted_name": "Tiwa, Southern",
+      "name": "Southern Tiwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tiy",
+      "name": "Tiruray",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tiz",
+      "name": "Tai Hongjin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tja",
+      "name": "Tajuasohn",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tjg",
+      "name": "Tunjung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tji",
+      "inverted_name": "Tujia, Northern",
+      "name": "Northern Tujia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tjj",
+      "name": "Tjungundji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tjl",
+      "name": "Tai Laing",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tjm",
+      "name": "Timucua",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tjn",
+      "name": "Tonjon",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tjo",
+      "inverted_name": "Tamazight, Temacine",
+      "name": "Temacine Tamazight",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tjp",
+      "name": "Tjupany",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tjs",
+      "inverted_name": "Tujia, Southern",
+      "name": "Southern Tujia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tju",
+      "name": "Tjurruru",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tjw",
+      "name": "Djabwurrung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tka",
+      "name": "Truká",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tkb",
+      "name": "Buksa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tkd",
+      "name": "Tukudede",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tke",
+      "name": "Takwane",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tkf",
+      "name": "Tukumanféd",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tkg",
+      "inverted_name": "Malagasy, Tesaka",
+      "name": "Tesaka Malagasy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tkl",
+      "name": "Tokelau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tkm",
+      "name": "Takelma",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tkn",
+      "name": "Toku-No-Shima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tkp",
+      "name": "Tikopia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tkq",
+      "name": "Tee",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tkr",
+      "name": "Tsakhur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tks",
+      "name": "Takestani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tkt",
+      "inverted_name": "Tharu, Kathoriya",
+      "name": "Kathoriya Tharu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tku",
+      "inverted_name": "Totonac, Upper Necaxa",
+      "name": "Upper Necaxa Totonac",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tkv",
+      "name": "Mur Pano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tkw",
+      "name": "Teanu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tkx",
+      "name": "Tangko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tkz",
+      "name": "Takua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tla",
+      "inverted_name": "Tepehuan, Southwestern",
+      "name": "Southwestern Tepehuan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlb",
+      "name": "Tobelo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlc",
+      "inverted_name": "Totonac, Yecuatla",
+      "name": "Yecuatla Totonac",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tld",
+      "name": "Talaud",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlf",
+      "name": "Telefol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlg",
+      "name": "Tofanma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlh",
+      "name": "Klingon",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "tli",
+      "name": "Tlingit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlj",
+      "name": "Talinga-Bwisi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlk",
+      "name": "Taloki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tll",
+      "name": "Tetela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlm",
+      "name": "Tolomako",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tln",
+      "name": "Talondo'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlo",
+      "name": "Talodi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlp",
+      "inverted_name": "Totonac, Filomena Mata-Coahuitlán",
+      "name": "Filomena Mata-Coahuitlán Totonac",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlq",
+      "name": "Tai Loi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlr",
+      "name": "Talise",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tls",
+      "name": "Tambotalo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlt",
+      "name": "Sou Nama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlu",
+      "name": "Tulehu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlv",
+      "name": "Taliabu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tlx",
+      "name": "Khehek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tly",
+      "name": "Talysh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tma",
+      "name": "Tama (Chad)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmb",
+      "name": "Katbol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmc",
+      "name": "Tumak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmd",
+      "name": "Haruai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tme",
+      "name": "Tremembé",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tmf",
+      "name": "Toba-Maskoy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmg",
+      "name": "Ternateño",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tmh",
+      "name": "Tamashek",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmi",
+      "name": "Tutuba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmj",
+      "name": "Samarokena",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmk",
+      "inverted_name": "Tamang, Northwestern",
+      "name": "Northwestern Tamang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tml",
+      "inverted_name": "Citak, Tamnim",
+      "name": "Tamnim Citak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmm",
+      "name": "Tai Thanh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmn",
+      "name": "Taman (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmo",
+      "name": "Temoq",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmq",
+      "name": "Tumleo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmr",
+      "inverted_name": "Aramaic, Jewish Babylonian (ca. 200-1200 CE)",
+      "name": "Jewish Babylonian Aramaic (ca. 200-1200 CE)",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tms",
+      "name": "Tima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmt",
+      "name": "Tasmate",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmu",
+      "name": "Iau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmv",
+      "name": "Tembo (Motembo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmw",
+      "name": "Temuan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmy",
+      "name": "Tami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tmz",
+      "name": "Tamanaku",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tna",
+      "name": "Tacana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnb",
+      "inverted_name": "Tunebo, Western",
+      "name": "Western Tunebo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnc",
+      "name": "Tanimuca-Retuarã",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnd",
+      "inverted_name": "Tunebo, Angosturas",
+      "name": "Angosturas Tunebo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tng",
+      "name": "Tobanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnh",
+      "name": "Maiani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tni",
+      "name": "Tandia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnk",
+      "name": "Kwamera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnl",
+      "name": "Lenakel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnm",
+      "name": "Tabla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnn",
+      "inverted_name": "Tanna, North",
+      "name": "North Tanna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tno",
+      "name": "Toromono",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnp",
+      "name": "Whitesands",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnq",
+      "name": "Taino",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tnr",
+      "name": "Ménik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tns",
+      "name": "Tenis",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnt",
+      "name": "Tontemboan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnu",
+      "name": "Tay Khang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnv",
+      "name": "Tangchangya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnw",
+      "name": "Tonsawang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnx",
+      "name": "Tanema",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tny",
+      "name": "Tongwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tnz",
+      "name": "Ten'edn",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tob",
+      "name": "Toba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "toc",
+      "inverted_name": "Totonac, Coyutla",
+      "name": "Coyutla Totonac",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tod",
+      "name": "Toma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tof",
+      "name": "Gizrra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tog",
+      "name": "Tonga (Nyasa)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "toh",
+      "name": "Gitonga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "toi",
+      "name": "Tonga (Zambia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "toj",
+      "name": "Tojolabal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tok",
+      "name": "Toki Pona",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "tol",
+      "name": "Tolowa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tom",
+      "name": "Tombulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "to",
+      "alpha_3": "ton",
+      "name": "Tonga (Tonga Islands)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "too",
+      "inverted_name": "Totonac, Xicotepec De Juárez",
+      "name": "Xicotepec De Juárez Totonac",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "top",
+      "inverted_name": "Totonac, Papantla",
+      "name": "Papantla Totonac",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "toq",
+      "name": "Toposa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tor",
+      "inverted_name": "Banda, Togbo-Vara",
+      "name": "Togbo-Vara Banda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tos",
+      "inverted_name": "Totonac, Highland",
+      "name": "Highland Totonac",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tou",
+      "name": "Tho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tov",
+      "inverted_name": "Taromi, Upper",
+      "name": "Upper Taromi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tow",
+      "name": "Jemez",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tox",
+      "name": "Tobian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "toy",
+      "name": "Topoiyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "toz",
+      "name": "To",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpa",
+      "name": "Taupota",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpc",
+      "inverted_name": "Me'phaa, Azoyú",
+      "name": "Azoyú Me'phaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpe",
+      "name": "Tippera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpf",
+      "name": "Tarpia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpg",
+      "name": "Kula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpi",
+      "name": "Tok Pisin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpj",
+      "name": "Tapieté",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpk",
+      "name": "Tupinikin",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tpl",
+      "inverted_name": "Me'phaa, Tlacoapa",
+      "name": "Tlacoapa Me'phaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpm",
+      "name": "Tampulma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpn",
+      "name": "Tupinambá",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tpo",
+      "name": "Tai Pao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpp",
+      "inverted_name": "Tepehua, Pisaflores",
+      "name": "Pisaflores Tepehua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpq",
+      "name": "Tukpa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpr",
+      "name": "Tuparí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpt",
+      "inverted_name": "Tepehua, Tlachichilco",
+      "name": "Tlachichilco Tepehua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpu",
+      "name": "Tampuan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpv",
+      "name": "Tanapag",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpw",
+      "name": "Tupí",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tpx",
+      "inverted_name": "Me'phaa, Acatepec",
+      "name": "Acatepec Me'phaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpy",
+      "name": "Trumai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tpz",
+      "name": "Tinputz",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tqb",
+      "name": "Tembé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tql",
+      "name": "Lehali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tqm",
+      "name": "Turumsa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tqn",
+      "name": "Tenino",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tqo",
+      "name": "Toaripi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tqp",
+      "name": "Tomoip",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tqq",
+      "name": "Tunni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tqr",
+      "name": "Torona",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tqt",
+      "inverted_name": "Totonac, Western",
+      "name": "Western Totonac",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tqu",
+      "name": "Touo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tqw",
+      "name": "Tonkawa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tra",
+      "name": "Tirahi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trb",
+      "name": "Terebu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trc",
+      "inverted_name": "Triqui, Copala",
+      "name": "Copala Triqui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trd",
+      "name": "Turi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tre",
+      "inverted_name": "Tarangan, East",
+      "name": "East Tarangan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trf",
+      "inverted_name": "Creole English, Trinidadian",
+      "name": "Trinidadian Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trg",
+      "name": "Lishán Didán",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trh",
+      "name": "Turaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tri",
+      "name": "Trió",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trj",
+      "name": "Toram",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trl",
+      "inverted_name": "Scottish, Traveller",
+      "name": "Traveller Scottish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trm",
+      "name": "Tregami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trn",
+      "name": "Trinitario",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tro",
+      "inverted_name": "Naga, Tarao",
+      "name": "Tarao Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trp",
+      "name": "Kok Borok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trq",
+      "inverted_name": "Triqui, San Martín Itunyoso",
+      "name": "San Martín Itunyoso Triqui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trr",
+      "name": "Taushiro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trs",
+      "inverted_name": "Triqui, Chicahuaxtla",
+      "name": "Chicahuaxtla Triqui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trt",
+      "name": "Tunggare",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tru",
+      "name": "Turoyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trv",
+      "name": "Sediq",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trw",
+      "name": "Torwali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "trx",
+      "inverted_name": "Bidayuh, Tringgus-Sembaan",
+      "name": "Tringgus-Sembaan Bidayuh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "try",
+      "name": "Turung",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "trz",
+      "name": "Torá",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tsa",
+      "name": "Tsaangi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsb",
+      "name": "Tsamai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsc",
+      "name": "Tswa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsd",
+      "name": "Tsakonian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tse",
+      "name": "Tunisian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsg",
+      "name": "Tausug",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsh",
+      "name": "Tsuvan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsi",
+      "name": "Tsimshian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsj",
+      "name": "Tshangla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsk",
+      "name": "Tseku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsl",
+      "name": "Ts'ün-Lao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsm",
+      "name": "Turkish Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "tn",
+      "alpha_3": "tsn",
+      "name": "Tswana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ts",
+      "alpha_3": "tso",
+      "name": "Tsonga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsp",
+      "inverted_name": "Toussian, Northern",
+      "name": "Northern Toussian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsq",
+      "name": "Thai Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsr",
+      "name": "Akei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tss",
+      "name": "Taiwan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tst",
+      "inverted_name": "Songway Kiini, Tondi",
+      "name": "Tondi Songway Kiini",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsu",
+      "name": "Tsou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsv",
+      "name": "Tsogo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsw",
+      "name": "Tsishingini",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsx",
+      "name": "Mubami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsy",
+      "name": "Tebul Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tsz",
+      "name": "Purepecha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tta",
+      "name": "Tutelo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ttb",
+      "name": "Gaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttc",
+      "name": "Tektiteko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttd",
+      "name": "Tauade",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tte",
+      "name": "Bwanabwana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttf",
+      "name": "Tuotomb",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttg",
+      "name": "Tutong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tth",
+      "inverted_name": "Ta'oih, Upper",
+      "name": "Upper Ta'oih",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tti",
+      "name": "Tobati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttj",
+      "name": "Tooro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttk",
+      "name": "Totoro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttl",
+      "name": "Totela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttm",
+      "inverted_name": "Tutchone, Northern",
+      "name": "Northern Tutchone",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttn",
+      "name": "Towei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tto",
+      "inverted_name": "Ta'oih, Lower",
+      "name": "Lower Ta'oih",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttp",
+      "name": "Tombelala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttq",
+      "inverted_name": "Tamajaq, Tawallammat",
+      "name": "Tawallammat Tamajaq",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttr",
+      "name": "Tera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tts",
+      "inverted_name": "Thai, Northeastern",
+      "name": "Northeastern Thai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttt",
+      "inverted_name": "Tat, Muslim",
+      "name": "Muslim Tat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttu",
+      "name": "Torau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttv",
+      "name": "Titan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttw",
+      "name": "Long Wat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tty",
+      "name": "Sikaritai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ttz",
+      "name": "Tsum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tua",
+      "name": "Wiarumus",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tub",
+      "name": "Tübatulabal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tuc",
+      "name": "Mutu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tud",
+      "name": "Tuxá",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tue",
+      "name": "Tuyuca",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tuf",
+      "inverted_name": "Tunebo, Central",
+      "name": "Central Tunebo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tug",
+      "name": "Tunia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tuh",
+      "name": "Taulil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tui",
+      "name": "Tupuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tuj",
+      "name": "Tugutil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "tk",
+      "alpha_3": "tuk",
+      "name": "Turkmen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tul",
+      "name": "Tula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tum",
+      "name": "Tumbuka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tun",
+      "name": "Tunica",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tuo",
+      "name": "Tucano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tuq",
+      "name": "Tedaga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "tr",
+      "alpha_3": "tur",
+      "name": "Turkish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tus",
+      "name": "Tuscarora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tuu",
+      "name": "Tututni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tuv",
+      "name": "Turkana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tux",
+      "name": "Tuxináwa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tuy",
+      "name": "Tugen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tuz",
+      "name": "Turka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tva",
+      "name": "Vaghua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tvd",
+      "name": "Tsuvadi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tve",
+      "name": "Te'un",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tvk",
+      "inverted_name": "Ambrym, Southeast",
+      "name": "Southeast Ambrym",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tvl",
+      "name": "Tuvalu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tvm",
+      "name": "Tela-Masbuar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tvn",
+      "name": "Tavoyan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tvo",
+      "name": "Tidore",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tvs",
+      "name": "Taveta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tvt",
+      "inverted_name": "Naga, Tutsa",
+      "name": "Tutsa Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tvu",
+      "name": "Tunen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tvw",
+      "name": "Sedoa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tvx",
+      "name": "Taivoan",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tvy",
+      "inverted_name": "Pidgin, Timor",
+      "name": "Timor Pidgin",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "twa",
+      "name": "Twana",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "twb",
+      "inverted_name": "Tawbuid, Western",
+      "name": "Western Tawbuid",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twc",
+      "name": "Teshenawa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "twd",
+      "name": "Twents",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twe",
+      "name": "Tewa (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twf",
+      "inverted_name": "Tiwa, Northern",
+      "name": "Northern Tiwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twg",
+      "name": "Tereweng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twh",
+      "name": "Tai Dón",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "tw",
+      "alpha_3": "twi",
+      "name": "Twi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twl",
+      "name": "Tawara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twm",
+      "inverted_name": "Monpa, Tawang",
+      "name": "Tawang Monpa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twn",
+      "name": "Twendi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "two",
+      "name": "Tswapong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twp",
+      "name": "Ere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twq",
+      "name": "Tasawaq",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twr",
+      "inverted_name": "Tarahumara, Southwestern",
+      "name": "Southwestern Tarahumara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twt",
+      "name": "Turiwára",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "twu",
+      "name": "Termanu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tww",
+      "name": "Tuwari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twx",
+      "name": "Tewe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "twy",
+      "name": "Tawoyan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "txa",
+      "name": "Tombonuo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "txb",
+      "name": "Tokharian B",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "txc",
+      "name": "Tsetsaut",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "txe",
+      "name": "Totoli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "txg",
+      "name": "Tangut",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "txh",
+      "name": "Thracian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "txi",
+      "name": "Ikpeng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "txj",
+      "name": "Tarjumo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "txm",
+      "name": "Tomini",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "txn",
+      "inverted_name": "Tarangan, West",
+      "name": "West Tarangan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "txo",
+      "name": "Toto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "txq",
+      "name": "Tii",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "txr",
+      "name": "Tartessian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "txs",
+      "name": "Tonsea",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "txt",
+      "name": "Citak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "txu",
+      "name": "Kayapó",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "txx",
+      "name": "Tatana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "txy",
+      "inverted_name": "Malagasy, Tanosy",
+      "name": "Tanosy Malagasy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tya",
+      "name": "Tauya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tye",
+      "name": "Kyanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tyh",
+      "name": "O'du",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tyi",
+      "name": "Teke-Tsaayi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tyj",
+      "name": "Tai Do",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tyl",
+      "name": "Thu Lao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tyn",
+      "name": "Kombai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "typ",
+      "name": "Thaypan",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "tyr",
+      "name": "Tai Daeng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tys",
+      "name": "Tày Sa Pa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tyt",
+      "name": "Tày Tac",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tyu",
+      "name": "Kua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tyv",
+      "name": "Tuvinian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tyx",
+      "name": "Teke-Tyee",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tyy",
+      "name": "Tiyaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tyz",
+      "name": "Tày",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tza",
+      "name": "Tanzanian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tzh",
+      "name": "Tzeltal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tzj",
+      "name": "Tz'utujil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tzl",
+      "name": "Talossan",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "tzm",
+      "inverted_name": "Tamazight, Central Atlas",
+      "name": "Central Atlas Tamazight",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tzn",
+      "name": "Tugun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tzo",
+      "name": "Tzotzil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "tzx",
+      "name": "Tabriak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uam",
+      "name": "Uamué",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "uan",
+      "name": "Kuan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uar",
+      "name": "Tairuma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uba",
+      "name": "Ubang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ubi",
+      "name": "Ubi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ubl",
+      "inverted_name": "Bikol, Buhi'non",
+      "name": "Buhi'non Bikol",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ubr",
+      "name": "Ubir",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ubu",
+      "name": "Umbu-Ungu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uby",
+      "name": "Ubykh",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "uda",
+      "name": "Uda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ude",
+      "name": "Udihe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "udg",
+      "name": "Muduga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "udi",
+      "name": "Udi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "udj",
+      "name": "Ujir",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "udl",
+      "name": "Wuzlam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "udm",
+      "name": "Udmurt",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "udu",
+      "name": "Uduk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ues",
+      "name": "Kioko",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ufi",
+      "name": "Ufim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uga",
+      "name": "Ugaritic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "ugb",
+      "name": "Kuku-Ugbanh",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "uge",
+      "name": "Ughele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ugh",
+      "name": "Kubachi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ugn",
+      "name": "Ugandan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ugo",
+      "name": "Ugong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ugy",
+      "name": "Uruguayan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uha",
+      "name": "Uhami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uhn",
+      "name": "Damal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ug",
+      "alpha_3": "uig",
+      "name": "Uighur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uis",
+      "name": "Uisai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uiv",
+      "name": "Iyive",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uji",
+      "name": "Tanjijili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uka",
+      "name": "Kaburi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ukg",
+      "name": "Ukuriguma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ukh",
+      "name": "Ukhwejo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uki",
+      "name": "Kui (India)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ukk",
+      "name": "Muak Sa-aak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ukl",
+      "name": "Ukrainian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ukp",
+      "name": "Ukpe-Bayobiri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ukq",
+      "name": "Ukwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "uk",
+      "alpha_3": "ukr",
+      "name": "Ukrainian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uks",
+      "name": "Urubú-Kaapor Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uku",
+      "name": "Ukue",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ukv",
+      "name": "Kuku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ukw",
+      "name": "Ukwuani-Aboh-Ndoni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uky",
+      "name": "Kuuk-Yak",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ula",
+      "name": "Fungwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ulb",
+      "name": "Ulukwumi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ulc",
+      "name": "Ulch",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ule",
+      "name": "Lule",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ulf",
+      "name": "Usku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uli",
+      "name": "Ulithian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ulk",
+      "name": "Meriam Mir",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ull",
+      "name": "Ullatan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ulm",
+      "name": "Ulumanda'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uln",
+      "name": "Unserdeutsch",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ulu",
+      "name": "Uma' Lung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ulw",
+      "name": "Ulwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uma",
+      "name": "Umatilla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "umb",
+      "name": "Umbundu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "umc",
+      "name": "Marrucinian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "umd",
+      "name": "Umbindhamu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "umg",
+      "name": "Morrobalama",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "umi",
+      "name": "Ukit",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "umm",
+      "name": "Umon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "umn",
+      "inverted_name": "Naga, Makyan",
+      "name": "Makyan Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "umo",
+      "name": "Umotína",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ump",
+      "name": "Umpila",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "umr",
+      "name": "Umbugarla",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ums",
+      "name": "Pendau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "umu",
+      "name": "Munsee",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "una",
+      "inverted_name": "Watut, North",
+      "name": "North Watut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "und",
+      "name": "Undetermined",
+      "scope": "S",
+      "type": "S"
+    },
+    {
+      "alpha_3": "une",
+      "name": "Uneme",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ung",
+      "name": "Ngarinyin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uni",
+      "name": "Uni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "unk",
+      "name": "Enawené-Nawé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "unm",
+      "name": "Unami",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "unn",
+      "name": "Kurnai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "unr",
+      "name": "Mundari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "unu",
+      "name": "Unubahe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "unx",
+      "name": "Munda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "unz",
+      "inverted_name": "Kaili, Unde",
+      "name": "Unde Kaili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uon",
+      "name": "Kulon",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "upi",
+      "name": "Umeda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "upv",
+      "name": "Uripiv-Wala-Rano-Atchin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ura",
+      "name": "Urarina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "urb",
+      "name": "Urubú-Kaapor",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "urc",
+      "name": "Urningangg",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_2": "ur",
+      "alpha_3": "urd",
+      "name": "Urdu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ure",
+      "name": "Uru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "urf",
+      "name": "Uradhi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "urg",
+      "name": "Urigina",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "urh",
+      "name": "Urhobo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uri",
+      "name": "Urim",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "urk",
+      "name": "Urak Lawoi'",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "url",
+      "name": "Urali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "urm",
+      "name": "Urapmin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "urn",
+      "name": "Uruangnirin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uro",
+      "name": "Ura (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "urp",
+      "name": "Uru-Pa-In",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "urr",
+      "name": "Lehalurup",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "urt",
+      "name": "Urat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uru",
+      "name": "Urumi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "urv",
+      "name": "Uruava",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "urw",
+      "name": "Sop",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "urx",
+      "name": "Urimo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ury",
+      "name": "Orya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "urz",
+      "name": "Uru-Eu-Wau-Wau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "usa",
+      "name": "Usarufa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ush",
+      "name": "Ushojo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "usi",
+      "name": "Usui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "usk",
+      "name": "Usaghade",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "usp",
+      "name": "Uspanteco",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uss",
+      "name": "us-Saare",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "usu",
+      "name": "Uya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uta",
+      "name": "Otank",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ute",
+      "name": "Ute-Southern Paiute",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uth",
+      "name": "ut-Hun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "utp",
+      "name": "Amba (Solomon Islands)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "utr",
+      "name": "Etulo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "utu",
+      "name": "Utu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uum",
+      "name": "Urum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uur",
+      "name": "Ura (Vanuatu)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uuu",
+      "name": "U",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uve",
+      "inverted_name": "Uvean, West",
+      "name": "West Uvean",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uvh",
+      "name": "Uri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uvl",
+      "name": "Lote",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uwa",
+      "name": "Kuku-Uwanh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uya",
+      "name": "Doko-Uyanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "uz",
+      "alpha_3": "uzb",
+      "name": "Uzbek",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uzn",
+      "inverted_name": "Uzbek, Northern",
+      "name": "Northern Uzbek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "uzs",
+      "inverted_name": "Uzbek, Southern",
+      "name": "Southern Uzbek",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vaa",
+      "name": "Vaagri Booli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vae",
+      "name": "Vale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vaf",
+      "name": "Vafsi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vag",
+      "name": "Vagla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vah",
+      "name": "Varhadi-Nagpuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vai",
+      "name": "Vai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vaj",
+      "name": "Sekele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "val",
+      "name": "Vehes",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vam",
+      "name": "Vanimo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "van",
+      "name": "Valman",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vao",
+      "name": "Vao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vap",
+      "name": "Vaiphei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "var",
+      "name": "Huarijio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vas",
+      "name": "Vasavi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vau",
+      "name": "Vanuma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vav",
+      "name": "Varli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vay",
+      "name": "Wayu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vbb",
+      "inverted_name": "Babar, Southeast",
+      "name": "Southeast Babar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vbk",
+      "inverted_name": "Bontok, Southwestern",
+      "name": "Southwestern Bontok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vec",
+      "name": "Venetian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ved",
+      "name": "Veddah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vel",
+      "name": "Veluws",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vem",
+      "name": "Vemgo-Mabas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "ve",
+      "alpha_3": "ven",
+      "name": "Venda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "veo",
+      "name": "Ventureño",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "vep",
+      "name": "Veps",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ver",
+      "name": "Mom Jango",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vgr",
+      "name": "Vaghri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vgt",
+      "name": "Vlaamse Gebarentaal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vic",
+      "inverted_name": "Creole English, Virgin Islands",
+      "name": "Virgin Islands Creole English",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vid",
+      "name": "Vidunda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "vi",
+      "alpha_3": "vie",
+      "name": "Vietnamese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vif",
+      "name": "Vili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vig",
+      "name": "Viemo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vil",
+      "name": "Vilela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vin",
+      "name": "Vinza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vis",
+      "name": "Vishavan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vit",
+      "name": "Viti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "viv",
+      "name": "Iduna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vka",
+      "name": "Kariyarra",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "vkj",
+      "name": "Kujarge",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vkk",
+      "name": "Kaur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vkl",
+      "name": "Kulisusu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vkm",
+      "name": "Kamakan",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "vkn",
+      "name": "Koro Nulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vko",
+      "name": "Kodeoha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vkp",
+      "inverted_name": "Creole Portuguese, Korlai",
+      "name": "Korlai Creole Portuguese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vkt",
+      "inverted_name": "Malay, Tenggarong Kutai",
+      "name": "Tenggarong Kutai Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vku",
+      "name": "Kurrama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vkz",
+      "name": "Koro Zuba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vlp",
+      "name": "Valpei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vls",
+      "name": "Vlaams",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vma",
+      "name": "Martuyhunira",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vmb",
+      "name": "Barbaram",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "vmc",
+      "inverted_name": "Mixtec, Juxtlahuaca",
+      "name": "Juxtlahuaca Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vmd",
+      "inverted_name": "Koraga, Mudu",
+      "name": "Mudu Koraga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vme",
+      "inverted_name": "Masela, East",
+      "name": "East Masela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vmf",
+      "name": "Mainfränkisch",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vmg",
+      "name": "Lungalunga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vmh",
+      "name": "Maraghei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vmi",
+      "name": "Miwa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "vmj",
+      "inverted_name": "Mixtec, Ixtayutla",
+      "name": "Ixtayutla Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vmk",
+      "name": "Makhuwa-Shirima",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vml",
+      "name": "Malgana",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "vmm",
+      "inverted_name": "Mixtec, Mitlatongo",
+      "name": "Mitlatongo Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vmp",
+      "inverted_name": "Mazatec, Soyaltepec",
+      "name": "Soyaltepec Mazatec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vmq",
+      "inverted_name": "Mixtec, Soyaltepec",
+      "name": "Soyaltepec Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vmr",
+      "name": "Marenje",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vms",
+      "name": "Moksela",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "vmu",
+      "name": "Muluridyi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "vmv",
+      "inverted_name": "Maidu, Valley",
+      "name": "Valley Maidu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "vmw",
+      "name": "Makhuwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vmx",
+      "inverted_name": "Mixtec, Tamazola",
+      "name": "Tamazola Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vmy",
+      "inverted_name": "Mazatec, Ayautla",
+      "name": "Ayautla Mazatec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vmz",
+      "inverted_name": "Mazatec, Mazatlán",
+      "name": "Mazatlán Mazatec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vnk",
+      "name": "Vano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vnm",
+      "name": "Vinmavis",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vnp",
+      "name": "Vunapu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "vo",
+      "alpha_3": "vol",
+      "name": "Volapük",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "vor",
+      "name": "Voro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vot",
+      "name": "Votic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vra",
+      "name": "Vera'a",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vro",
+      "name": "Võro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vrs",
+      "name": "Varisi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vrt",
+      "name": "Burmbar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vsi",
+      "name": "Moldova Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vsl",
+      "name": "Venezuelan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vsv",
+      "name": "Valencian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vto",
+      "name": "Vitou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vum",
+      "name": "Vumbu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vun",
+      "name": "Vunjo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vut",
+      "name": "Vute",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "vwa",
+      "name": "Awa (China)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "waa",
+      "name": "Walla Walla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wab",
+      "name": "Wab",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wac",
+      "name": "Wasco-Wishram",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wad",
+      "name": "Wamesa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wae",
+      "name": "Walser",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "waf",
+      "name": "Wakoná",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wag",
+      "name": "Wa'ema",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wah",
+      "name": "Watubela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wai",
+      "name": "Wares",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "waj",
+      "name": "Waffa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wal",
+      "name": "Wolaytta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wam",
+      "name": "Wampanoag",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wan",
+      "name": "Wan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wao",
+      "name": "Wappo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wap",
+      "name": "Wapishana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "waq",
+      "name": "Wagiman",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "war",
+      "name": "Waray (Philippines)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "was",
+      "name": "Washo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wat",
+      "name": "Kaninuwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wau",
+      "name": "Waurá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wav",
+      "name": "Waka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "waw",
+      "name": "Waiwai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wax",
+      "name": "Watam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "way",
+      "name": "Wayana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "waz",
+      "name": "Wampur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wba",
+      "name": "Warao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbb",
+      "name": "Wabo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbe",
+      "name": "Waritai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbf",
+      "name": "Wara",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbh",
+      "name": "Wanda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbi",
+      "name": "Vwanji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbj",
+      "name": "Alagwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbk",
+      "name": "Waigali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbl",
+      "name": "Wakhi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbm",
+      "name": "Wa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbp",
+      "name": "Warlpiri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbq",
+      "name": "Waddar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbr",
+      "name": "Wagdi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbs",
+      "name": "West Bengal Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbt",
+      "name": "Warnman",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbv",
+      "name": "Wajarri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wbw",
+      "name": "Woi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wca",
+      "name": "Yanomámi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wci",
+      "inverted_name": "Gbe, Waci",
+      "name": "Waci Gbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wdd",
+      "name": "Wandji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wdg",
+      "name": "Wadaginam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wdj",
+      "name": "Wadjiginy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wdk",
+      "name": "Wadikali",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wdt",
+      "name": "Wendat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wdu",
+      "name": "Wadjigu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wdy",
+      "name": "Wadjabangayi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wea",
+      "name": "Wewaw",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wec",
+      "name": "Wè Western",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wed",
+      "name": "Wedau",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "weg",
+      "name": "Wergaia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "weh",
+      "name": "Weh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wei",
+      "name": "Kiunum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wem",
+      "inverted_name": "Gbe, Weme",
+      "name": "Weme Gbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "weo",
+      "name": "Wemale",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wep",
+      "name": "Westphalien",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wer",
+      "name": "Weri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wes",
+      "inverted_name": "Pidgin, Cameroon",
+      "name": "Cameroon Pidgin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wet",
+      "name": "Perai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "weu",
+      "inverted_name": "Chin, Rawngtu",
+      "name": "Rawngtu Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wew",
+      "name": "Wejewa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wfg",
+      "name": "Yafi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wga",
+      "name": "Wagaya",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wgb",
+      "name": "Wagawaga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wgg",
+      "name": "Wangkangurru",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wgi",
+      "name": "Wahgi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wgo",
+      "name": "Waigeo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wgu",
+      "name": "Wirangu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wgy",
+      "name": "Warrgamay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wha",
+      "name": "Sou Upaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "whg",
+      "inverted_name": "Wahgi, North",
+      "name": "North Wahgi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "whk",
+      "inverted_name": "Kenyah, Wahau",
+      "name": "Wahau Kenyah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "whu",
+      "inverted_name": "Kayan, Wahau",
+      "name": "Wahau Kayan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wib",
+      "inverted_name": "Toussian, Southern",
+      "name": "Southern Toussian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wic",
+      "name": "Wichita",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wie",
+      "name": "Wik-Epa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wif",
+      "name": "Wik-Keyangan",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wig",
+      "name": "Wik Ngathan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wih",
+      "name": "Wik-Me'anha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wii",
+      "name": "Minidien",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wij",
+      "name": "Wik-Iiyanh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wik",
+      "name": "Wikalkan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wil",
+      "name": "Wilawila",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wim",
+      "name": "Wik-Mungkan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "win",
+      "name": "Ho-Chunk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wir",
+      "name": "Wiraféd",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wiu",
+      "name": "Wiru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wiv",
+      "name": "Vitu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wiy",
+      "name": "Wiyot",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wja",
+      "name": "Waja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wji",
+      "name": "Warji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wka",
+      "name": "Kw'adza",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wkb",
+      "name": "Kumbaran",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wkd",
+      "name": "Wakde",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wkl",
+      "name": "Kalanadi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wkr",
+      "name": "Keerray-Woorroong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wku",
+      "name": "Kunduvadi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wkw",
+      "name": "Wakawaka",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wky",
+      "name": "Wangkayutyuru",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wla",
+      "name": "Walio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wlc",
+      "inverted_name": "Comorian, Mwali",
+      "name": "Mwali Comorian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wle",
+      "name": "Wolane",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wlg",
+      "name": "Kunbarlang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wlh",
+      "name": "Welaun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wli",
+      "name": "Waioli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wlk",
+      "name": "Wailaki",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wll",
+      "name": "Wali (Sudan)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wlm",
+      "inverted_name": "Welsh, Middle",
+      "name": "Middle Welsh",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_2": "wa",
+      "alpha_3": "wln",
+      "name": "Walloon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wlo",
+      "name": "Wolio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wlr",
+      "name": "Wailapa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wls",
+      "name": "Wallisian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wlu",
+      "name": "Wuliwuli",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wlv",
+      "name": "Wichí Lhamtés Vejoz",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wlw",
+      "name": "Walak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wlx",
+      "name": "Wali (Ghana)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wly",
+      "name": "Waling",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wma",
+      "name": "Mawa (Nigeria)",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wmb",
+      "name": "Wambaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wmc",
+      "name": "Wamas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wmd",
+      "name": "Mamaindé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wme",
+      "name": "Wambule",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wmg",
+      "inverted_name": "Minyag, Western",
+      "name": "Western Minyag",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wmh",
+      "name": "Waima'a",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wmi",
+      "name": "Wamin",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wmm",
+      "name": "Maiwa (Indonesia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wmn",
+      "name": "Waamwang",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wmo",
+      "name": "Wom (Papua New Guinea)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wms",
+      "name": "Wambon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wmt",
+      "name": "Walmajarri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wmw",
+      "name": "Mwani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wmx",
+      "name": "Womo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wnb",
+      "name": "Wanambre",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wnc",
+      "name": "Wantoat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wnd",
+      "name": "Wandarang",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wne",
+      "name": "Waneci",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wng",
+      "name": "Wanggom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wni",
+      "inverted_name": "Comorian, Ndzwani",
+      "name": "Ndzwani Comorian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wnk",
+      "name": "Wanukaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wnm",
+      "name": "Wanggamala",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wnn",
+      "name": "Wunumara",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wno",
+      "name": "Wano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wnp",
+      "name": "Wanap",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wnu",
+      "name": "Usan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wnw",
+      "name": "Wintu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wny",
+      "name": "Wanyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "woa",
+      "name": "Kuwema",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wob",
+      "name": "Wè Northern",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "woc",
+      "name": "Wogeo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wod",
+      "name": "Wolani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "woe",
+      "name": "Woleaian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wof",
+      "inverted_name": "Wolof, Gambian",
+      "name": "Gambian Wolof",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wog",
+      "name": "Wogamusin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "woi",
+      "name": "Kamang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wok",
+      "name": "Longto",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "wo",
+      "alpha_3": "wol",
+      "name": "Wolof",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wom",
+      "name": "Wom (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "won",
+      "name": "Wongo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "woo",
+      "name": "Manombai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wor",
+      "name": "Woria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wos",
+      "name": "Hanga Hundi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wow",
+      "name": "Wawonii",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "woy",
+      "name": "Weyto",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wpc",
+      "name": "Maco",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wrb",
+      "name": "Waluwarra",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wrg",
+      "name": "Warungu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wrh",
+      "name": "Wiradjuri",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wri",
+      "name": "Wariyangga",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wrk",
+      "name": "Garrwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wrl",
+      "name": "Warlmanpa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wrm",
+      "name": "Warumungu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wrn",
+      "name": "Warnang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wro",
+      "name": "Worrorra",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wrp",
+      "name": "Waropen",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wrr",
+      "name": "Wardaman",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wrs",
+      "name": "Waris",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wru",
+      "name": "Waru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wrv",
+      "name": "Waruna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wrw",
+      "name": "Gugu Warra",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wrx",
+      "name": "Wae Rana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wry",
+      "name": "Merwari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wrz",
+      "name": "Waray (Australia)",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wsa",
+      "name": "Warembori",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wsg",
+      "inverted_name": "Gondi, Adilabad",
+      "name": "Adilabad Gondi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wsi",
+      "name": "Wusi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wsk",
+      "name": "Waskia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wsr",
+      "name": "Owenia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wss",
+      "name": "Wasa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wsu",
+      "name": "Wasu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wsv",
+      "name": "Wotapuri-Katarqalai",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wtf",
+      "name": "Watiwa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wth",
+      "name": "Wathawurrung",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wti",
+      "name": "Berta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wtk",
+      "name": "Watakataui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wtm",
+      "name": "Mewati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wtw",
+      "name": "Wotu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wua",
+      "name": "Wikngenchera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wub",
+      "name": "Wunambal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wud",
+      "name": "Wudu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wuh",
+      "name": "Wutunhua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wul",
+      "name": "Silimo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wum",
+      "name": "Wumbvu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wun",
+      "name": "Bungu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wur",
+      "name": "Wurrugu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wut",
+      "name": "Wutung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wuu",
+      "inverted_name": "Chinese, Wu",
+      "name": "Wu Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wuv",
+      "name": "Wuvulu-Aua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wux",
+      "name": "Wulna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wuy",
+      "name": "Wauyai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wwa",
+      "name": "Waama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wwb",
+      "name": "Wakabunga",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wwo",
+      "name": "Wetamut",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wwr",
+      "name": "Warrwa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "www",
+      "name": "Wawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wxa",
+      "name": "Waxianghua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wxw",
+      "name": "Wardandi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wyb",
+      "name": "Wangaaybuwan-Ngiyambaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wyi",
+      "name": "Woiwurrung",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "wym",
+      "name": "Wymysorys",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wyn",
+      "name": "Wyandot",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wyr",
+      "name": "Wayoró",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "wyy",
+      "inverted_name": "Fijian, Western",
+      "name": "Western Fijian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xaa",
+      "inverted_name": "Arabic, Andalusian",
+      "name": "Andalusian Arabic",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xab",
+      "name": "Sambe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xac",
+      "name": "Kachari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xad",
+      "name": "Adai",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xae",
+      "name": "Aequian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xag",
+      "name": "Aghwan",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xai",
+      "name": "Kaimbé",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xaj",
+      "name": "Ararandewára",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xak",
+      "name": "Máku",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xal",
+      "name": "Kalmyk",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xam",
+      "name": "ǀXam",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xan",
+      "name": "Xamtanga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xao",
+      "name": "Khao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xap",
+      "name": "Apalachee",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xaq",
+      "name": "Aquitanian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xar",
+      "name": "Karami",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xas",
+      "name": "Kamas",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xat",
+      "name": "Katawixi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xau",
+      "name": "Kauwera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xav",
+      "name": "Xavánte",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xaw",
+      "name": "Kawaiisu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xay",
+      "name": "Kayan Mahakam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xbb",
+      "inverted_name": "Burdekin, Lower",
+      "name": "Lower Burdekin",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xbc",
+      "name": "Bactrian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xbd",
+      "name": "Bindal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xbe",
+      "name": "Bigambal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xbg",
+      "name": "Bunganditj",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xbi",
+      "name": "Kombio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xbj",
+      "name": "Birrpayi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xbm",
+      "inverted_name": "Breton, Middle",
+      "name": "Middle Breton",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xbn",
+      "name": "Kenaboi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xbo",
+      "name": "Bolgarian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xbp",
+      "name": "Bibbulman",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xbr",
+      "name": "Kambera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xbw",
+      "name": "Kambiwá",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xby",
+      "name": "Batjala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xcb",
+      "name": "Cumbric",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xcc",
+      "name": "Camunic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xce",
+      "name": "Celtiberian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xcg",
+      "inverted_name": "Gaulish, Cisalpine",
+      "name": "Cisalpine Gaulish",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xch",
+      "name": "Chemakum",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xcl",
+      "inverted_name": "Armenian, Classical",
+      "name": "Classical Armenian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xcm",
+      "name": "Comecrudo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xcn",
+      "name": "Cotoname",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xco",
+      "name": "Chorasmian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xcr",
+      "name": "Carian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xct",
+      "inverted_name": "Tibetan, Classical",
+      "name": "Classical Tibetan",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xcu",
+      "name": "Curonian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xcv",
+      "name": "Chuvantsy",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xcw",
+      "name": "Coahuilteco",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xcy",
+      "name": "Cayuse",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xda",
+      "name": "Darkinyung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xdc",
+      "name": "Dacian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xdk",
+      "name": "Dharuk",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xdm",
+      "name": "Edomite",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xdo",
+      "name": "Kwandu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xdq",
+      "name": "Kaitag",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xdy",
+      "inverted_name": "Dayak, Malayic",
+      "name": "Malayic Dayak",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xeb",
+      "name": "Eblan",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xed",
+      "name": "Hdi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xeg",
+      "name": "ǁXegwi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xel",
+      "name": "Kelo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xem",
+      "name": "Kembayan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xep",
+      "name": "Epi-Olmec",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xer",
+      "name": "Xerénte",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xes",
+      "name": "Kesawai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xet",
+      "name": "Xetá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xeu",
+      "name": "Keoru-Ahia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xfa",
+      "name": "Faliscan",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xga",
+      "name": "Galatian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xgb",
+      "name": "Gbin",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xgd",
+      "name": "Gudang",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xgf",
+      "name": "Gabrielino-Fernandeño",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xgg",
+      "name": "Goreng",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xgi",
+      "name": "Garingbal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xgl",
+      "name": "Galindan",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xgm",
+      "name": "Dharumbal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xgr",
+      "name": "Garza",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xgu",
+      "name": "Unggumi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xgw",
+      "name": "Guwa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xha",
+      "name": "Harami",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xhc",
+      "name": "Hunnic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xhd",
+      "name": "Hadrami",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xhe",
+      "name": "Khetrani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xhm",
+      "inverted_name": "Khmer, Middle (1400 to 1850 CE)",
+      "name": "Middle Khmer (1400 to 1850 CE)",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_2": "xh",
+      "alpha_3": "xho",
+      "name": "Xhosa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xhr",
+      "name": "Hernican",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xht",
+      "name": "Hattic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xhu",
+      "name": "Hurrian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xhv",
+      "name": "Khua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xib",
+      "name": "Iberian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xii",
+      "name": "Xiri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xil",
+      "name": "Illyrian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xin",
+      "name": "Xinca",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xir",
+      "name": "Xiriâna",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xis",
+      "name": "Kisan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xiv",
+      "name": "Indus Valley Language",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xiy",
+      "name": "Xipaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xjb",
+      "name": "Minjungbal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xjt",
+      "name": "Jaitmatang",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xka",
+      "name": "Kalkoti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkb",
+      "inverted_name": "Nago, Northern",
+      "name": "Northern Nago",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkc",
+      "name": "Kho'ini",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkd",
+      "inverted_name": "Kayan, Mendalam",
+      "name": "Mendalam Kayan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xke",
+      "name": "Kereho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkf",
+      "name": "Khengkha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkg",
+      "name": "Kagoro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xki",
+      "name": "Kenyan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkj",
+      "name": "Kajali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkk",
+      "name": "Kachok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkl",
+      "name": "Mainstream Kenyah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkn",
+      "inverted_name": "Kayan, Kayan River",
+      "name": "Kayan River Kayan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xko",
+      "name": "Kiorr",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkp",
+      "name": "Kabatei",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkq",
+      "name": "Koroni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkr",
+      "name": "Xakriabá",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xks",
+      "name": "Kumbewaha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkt",
+      "name": "Kantosi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xku",
+      "name": "Kaamba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkv",
+      "name": "Kgalagadi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkw",
+      "name": "Kembra",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkx",
+      "name": "Karore",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xky",
+      "name": "Uma' Lasan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xkz",
+      "name": "Kurtokha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xla",
+      "name": "Kamula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xlb",
+      "name": "Loup B",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xlc",
+      "name": "Lycian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xld",
+      "name": "Lydian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xle",
+      "name": "Lemnian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xlg",
+      "name": "Ligurian (Ancient)",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xli",
+      "name": "Liburnian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xln",
+      "name": "Alanic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xlo",
+      "name": "Loup A",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xlp",
+      "name": "Lepontic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xls",
+      "name": "Lusitanian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xlu",
+      "inverted_name": "Luwian, Cuneiform",
+      "name": "Cuneiform Luwian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xly",
+      "name": "Elymian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xma",
+      "name": "Mushungulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmb",
+      "name": "Mbonga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmc",
+      "name": "Makhuwa-Marrevone",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmd",
+      "name": "Mbudum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xme",
+      "name": "Median",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xmf",
+      "name": "Mingrelian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmg",
+      "name": "Mengaka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmh",
+      "name": "Kugu-Muminh",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmj",
+      "name": "Majera",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmk",
+      "inverted_name": "Macedonian, Ancient",
+      "name": "Ancient Macedonian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xml",
+      "name": "Malaysian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmm",
+      "inverted_name": "Malay, Manado",
+      "name": "Manado Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmn",
+      "inverted_name": "Persian, Manichaean Middle",
+      "name": "Manichaean Middle Persian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xmo",
+      "name": "Morerebi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmp",
+      "name": "Kuku-Mu'inh",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xmq",
+      "name": "Kuku-Mangk",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xmr",
+      "name": "Meroitic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xms",
+      "name": "Moroccan Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmt",
+      "name": "Matbat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmu",
+      "name": "Kamu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xmv",
+      "inverted_name": "Malagasy, Antankarana",
+      "name": "Antankarana Malagasy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmw",
+      "inverted_name": "Malagasy, Tsimihety",
+      "name": "Tsimihety Malagasy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmx",
+      "name": "Salawati",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmy",
+      "name": "Mayaguduna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xmz",
+      "name": "Mori Bawah",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xna",
+      "inverted_name": "North Arabian, Ancient",
+      "name": "Ancient North Arabian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xnb",
+      "name": "Kanakanabu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xng",
+      "inverted_name": "Mongolian, Middle",
+      "name": "Middle Mongolian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xnh",
+      "name": "Kuanhua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xni",
+      "name": "Ngarigu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xnj",
+      "name": "Ngoni (Tanzania)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xnk",
+      "name": "Nganakarti",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xnm",
+      "name": "Ngumbarl",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xnn",
+      "inverted_name": "Kankanay, Northern",
+      "name": "Northern Kankanay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xno",
+      "name": "Anglo-Norman",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xnq",
+      "name": "Ngoni (Mozambique)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xnr",
+      "name": "Kangri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xns",
+      "name": "Kanashi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xnt",
+      "name": "Narragansett",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xnu",
+      "name": "Nukunul",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xny",
+      "name": "Nyiyaparli",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xnz",
+      "name": "Kenzi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xoc",
+      "name": "O'chi'chi'",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xod",
+      "name": "Kokoda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xog",
+      "name": "Soga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xoi",
+      "name": "Kominimung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xok",
+      "name": "Xokleng",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xom",
+      "name": "Komo (Sudan)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xon",
+      "name": "Konkomba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xoo",
+      "name": "Xukurú",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xop",
+      "name": "Kopar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xor",
+      "name": "Korubo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xow",
+      "name": "Kowaki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xpa",
+      "name": "Pirriya",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpb",
+      "inverted_name": "Tasmanian, Northeastern",
+      "name": "Northeastern Tasmanian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpc",
+      "name": "Pecheneg",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xpd",
+      "inverted_name": "Tasmanian, Oyster Bay",
+      "name": "Oyster Bay Tasmanian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpe",
+      "inverted_name": "Kpelle, Liberia",
+      "name": "Liberia Kpelle",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xpf",
+      "inverted_name": "Tasmanian, Southeast",
+      "name": "Southeast Tasmanian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpg",
+      "name": "Phrygian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xph",
+      "inverted_name": "Tasmanian, North Midlands",
+      "name": "North Midlands Tasmanian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpi",
+      "name": "Pictish",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xpj",
+      "name": "Mpalitjanh",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpk",
+      "inverted_name": "Pano, Kulina",
+      "name": "Kulina Pano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xpl",
+      "inverted_name": "Tasmanian, Port Sorell",
+      "name": "Port Sorell Tasmanian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpm",
+      "name": "Pumpokol",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpn",
+      "name": "Kapinawá",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpo",
+      "name": "Pochutec",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpp",
+      "name": "Puyo-Paekche",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xpq",
+      "name": "Mohegan-Pequot",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpr",
+      "name": "Parthian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xps",
+      "name": "Pisidian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xpt",
+      "name": "Punthamara",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpu",
+      "name": "Punic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xpv",
+      "inverted_name": "Tasmanian, Northern",
+      "name": "Northern Tasmanian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpw",
+      "inverted_name": "Tasmanian, Northwestern",
+      "name": "Northwestern Tasmanian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpx",
+      "inverted_name": "Tasmanian, Southwestern",
+      "name": "Southwestern Tasmanian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xpy",
+      "name": "Puyo",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xpz",
+      "inverted_name": "Tasmanian, Bruny Island",
+      "name": "Bruny Island Tasmanian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xqa",
+      "name": "Karakhanid",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xqt",
+      "name": "Qatabanian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xra",
+      "name": "Krahô",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xrb",
+      "inverted_name": "Karaboro, Eastern",
+      "name": "Eastern Karaboro",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xrd",
+      "name": "Gundungurra",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xre",
+      "name": "Kreye",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xrg",
+      "name": "Minang",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xri",
+      "name": "Krikati-Timbira",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xrm",
+      "name": "Armazic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xrn",
+      "name": "Arin",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xrr",
+      "name": "Raetic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xrt",
+      "name": "Aranama-Tamique",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xru",
+      "name": "Marriammu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xrw",
+      "name": "Karawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xsa",
+      "name": "Sabaean",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xsb",
+      "name": "Sambal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xsc",
+      "name": "Scythian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xsd",
+      "name": "Sidetic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xse",
+      "name": "Sempan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xsh",
+      "name": "Shamang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xsi",
+      "name": "Sio",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xsj",
+      "name": "Subi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xsl",
+      "inverted_name": "Slavey, South",
+      "name": "South Slavey",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xsm",
+      "name": "Kasem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xsn",
+      "name": "Sanga (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xso",
+      "name": "Solano",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xsp",
+      "name": "Silopi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xsq",
+      "name": "Makhuwa-Saka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xsr",
+      "name": "Sherpa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xss",
+      "name": "Assan",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xsu",
+      "name": "Sanumá",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xsv",
+      "name": "Sudovian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xsy",
+      "name": "Saisiyat",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xta",
+      "inverted_name": "Mixtec, Alcozauca",
+      "name": "Alcozauca Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xtb",
+      "inverted_name": "Mixtec, Chazumba",
+      "name": "Chazumba Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xtc",
+      "name": "Katcha-Kadugli-Miri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xtd",
+      "inverted_name": "Mixtec, Diuxi-Tilantongo",
+      "name": "Diuxi-Tilantongo Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xte",
+      "name": "Ketengban",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xtg",
+      "inverted_name": "Gaulish, Transalpine",
+      "name": "Transalpine Gaulish",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xth",
+      "name": "Yitha Yitha",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xti",
+      "inverted_name": "Mixtec, Sinicahua",
+      "name": "Sinicahua Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xtj",
+      "inverted_name": "Mixtec, San Juan Teita",
+      "name": "San Juan Teita Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xtl",
+      "inverted_name": "Mixtec, Tijaltepec",
+      "name": "Tijaltepec Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xtm",
+      "inverted_name": "Mixtec, Magdalena Peñasco",
+      "name": "Magdalena Peñasco Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xtn",
+      "inverted_name": "Mixtec, Northern Tlaxiaco",
+      "name": "Northern Tlaxiaco Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xto",
+      "name": "Tokharian A",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xtp",
+      "inverted_name": "Mixtec, San Miguel Piedras",
+      "name": "San Miguel Piedras Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xtq",
+      "name": "Tumshuqese",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "xtr",
+      "inverted_name": "Tripuri, Early",
+      "name": "Early Tripuri",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xts",
+      "inverted_name": "Mixtec, Sindihui",
+      "name": "Sindihui Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xtt",
+      "inverted_name": "Mixtec, Tacahua",
+      "name": "Tacahua Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xtu",
+      "inverted_name": "Mixtec, Cuyamecalco",
+      "name": "Cuyamecalco Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xtv",
+      "name": "Thawa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xtw",
+      "name": "Tawandê",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xty",
+      "inverted_name": "Mixtec, Yoloxochitl",
+      "name": "Yoloxochitl Mixtec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xua",
+      "inverted_name": "Kurumba, Alu",
+      "name": "Alu Kurumba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xub",
+      "inverted_name": "Kurumba, Betta",
+      "name": "Betta Kurumba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xud",
+      "name": "Umiida",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xug",
+      "name": "Kunigami",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xuj",
+      "inverted_name": "Kurumba, Jennu",
+      "name": "Jennu Kurumba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xul",
+      "name": "Ngunawal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xum",
+      "name": "Umbrian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xun",
+      "name": "Unggaranggu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xuo",
+      "name": "Kuo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xup",
+      "inverted_name": "Umpqua, Upper",
+      "name": "Upper Umpqua",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xur",
+      "name": "Urartian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xut",
+      "name": "Kuthant",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xuu",
+      "name": "Kxoe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xve",
+      "name": "Venetic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xvi",
+      "name": "Kamviri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xvn",
+      "name": "Vandalic",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xvo",
+      "name": "Volscian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xvs",
+      "name": "Vestinian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xwa",
+      "name": "Kwaza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xwc",
+      "name": "Woccon",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xwd",
+      "name": "Wadi Wadi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xwe",
+      "inverted_name": "Gbe, Xwela",
+      "name": "Xwela Gbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xwg",
+      "name": "Kwegu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xwj",
+      "name": "Wajuk",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xwk",
+      "name": "Wangkumara",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xwl",
+      "inverted_name": "Gbe, Western Xwla",
+      "name": "Western Xwla Gbe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xwo",
+      "inverted_name": "Oirat, Written",
+      "name": "Written Oirat",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xwr",
+      "name": "Kwerba Mamberamo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xwt",
+      "name": "Wotjobaluk",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xww",
+      "name": "Wemba Wemba",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xxb",
+      "name": "Boro (Ghana)",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xxk",
+      "name": "Ke'o",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xxm",
+      "name": "Minkin",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xxr",
+      "name": "Koropó",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xxt",
+      "name": "Tambora",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xya",
+      "name": "Yaygir",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xyb",
+      "name": "Yandjibara",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xyj",
+      "name": "Mayi-Yapi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xyk",
+      "name": "Mayi-Kulan",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xyl",
+      "name": "Yalakalore",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xyt",
+      "name": "Mayi-Thakurti",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xyy",
+      "name": "Yorta Yorta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "xzh",
+      "name": "Zhang-Zhung",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "xzm",
+      "name": "Zemgalian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "xzp",
+      "inverted_name": "Zapotec, Ancient",
+      "name": "Ancient Zapotec",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "yaa",
+      "name": "Yaminahua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yab",
+      "name": "Yuhup",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yac",
+      "inverted_name": "Yali, Pass Valley",
+      "name": "Pass Valley Yali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yad",
+      "name": "Yagua",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yae",
+      "name": "Pumé",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yaf",
+      "name": "Yaka (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yag",
+      "name": "Yámana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yah",
+      "name": "Yazgulyam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yai",
+      "name": "Yagnobi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yaj",
+      "name": "Banda-Yangere",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yak",
+      "name": "Yakama",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yal",
+      "name": "Yalunka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yam",
+      "name": "Yamba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yan",
+      "name": "Mayangna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yao",
+      "name": "Yao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yap",
+      "name": "Yapese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yaq",
+      "name": "Yaqui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yar",
+      "name": "Yabarana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yas",
+      "name": "Nugunu (Cameroon)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yat",
+      "name": "Yambeta",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yau",
+      "name": "Yuwana",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yav",
+      "name": "Yangben",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yaw",
+      "name": "Yawalapití",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yax",
+      "name": "Yauma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yay",
+      "name": "Agwagwune",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yaz",
+      "name": "Lokaa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yba",
+      "name": "Yala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ybb",
+      "name": "Yemba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ybe",
+      "inverted_name": "Yugur, West",
+      "name": "West Yugur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ybh",
+      "name": "Yakha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ybi",
+      "name": "Yamphu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ybj",
+      "name": "Hasha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ybk",
+      "name": "Bokha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ybl",
+      "name": "Yukuben",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ybm",
+      "name": "Yaben",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ybn",
+      "name": "Yabaâna",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ybo",
+      "name": "Yabong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ybx",
+      "name": "Yawiyo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yby",
+      "name": "Yaweyuha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ych",
+      "name": "Chesu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ycl",
+      "name": "Lolopo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ycn",
+      "name": "Yucuna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ycp",
+      "name": "Chepya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yda",
+      "name": "Yanda",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ydd",
+      "inverted_name": "Yiddish, Eastern",
+      "name": "Eastern Yiddish",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yde",
+      "name": "Yangum Dey",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ydg",
+      "name": "Yidgha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ydk",
+      "name": "Yoidik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yea",
+      "name": "Ravula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yec",
+      "name": "Yeniche",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yee",
+      "name": "Yimas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yei",
+      "name": "Yeni",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yej",
+      "name": "Yevanic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yel",
+      "name": "Yela",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yer",
+      "name": "Tarok",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yes",
+      "name": "Nyankpa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yet",
+      "name": "Yetfa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yeu",
+      "name": "Yerukula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yev",
+      "name": "Yapunda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yey",
+      "name": "Yeyi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yga",
+      "name": "Malyangapa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ygi",
+      "name": "Yiningayi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ygl",
+      "name": "Yangum Gel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ygm",
+      "name": "Yagomi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ygp",
+      "name": "Gepo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ygr",
+      "name": "Yagaria",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ygs",
+      "name": "Yolŋu Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ygu",
+      "name": "Yugul",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ygw",
+      "name": "Yagwoia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yha",
+      "inverted_name": "Buyang, Baha",
+      "name": "Baha Buyang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yhd",
+      "inverted_name": "Arabic, Judeo-Iraqi",
+      "name": "Judeo-Iraqi Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yhl",
+      "inverted_name": "Phowa, Hlepho",
+      "name": "Hlepho Phowa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yhs",
+      "name": "Yan-nhaŋu Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yia",
+      "name": "Yinggarda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "yi",
+      "alpha_3": "yid",
+      "name": "Yiddish",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yif",
+      "name": "Ache",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yig",
+      "inverted_name": "Nasu, Wusa",
+      "name": "Wusa Nasu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yih",
+      "inverted_name": "Yiddish, Western",
+      "name": "Western Yiddish",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yii",
+      "name": "Yidiny",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yij",
+      "name": "Yindjibarndi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yik",
+      "inverted_name": "Lalo, Dongshanba",
+      "name": "Dongshanba Lalo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yil",
+      "name": "Yindjilandji",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yim",
+      "inverted_name": "Naga, Yimchungru",
+      "name": "Yimchungru Naga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yin",
+      "name": "Riang Lai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yip",
+      "name": "Pholo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yiq",
+      "name": "Miqie",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yir",
+      "inverted_name": "Awyu, North",
+      "name": "North Awyu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yis",
+      "name": "Yis",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yit",
+      "inverted_name": "Lalu, Eastern",
+      "name": "Eastern Lalu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yiu",
+      "name": "Awu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yiv",
+      "inverted_name": "Nisu, Northern",
+      "name": "Northern Nisu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yix",
+      "inverted_name": "Yi, Axi",
+      "name": "Axi Yi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yiz",
+      "name": "Azhe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yka",
+      "name": "Yakan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ykg",
+      "inverted_name": "Yukaghir, Northern",
+      "name": "Northern Yukaghir",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yki",
+      "name": "Yoke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ykk",
+      "name": "Yakaikeke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ykl",
+      "name": "Khlula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ykm",
+      "name": "Kap",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ykn",
+      "name": "Kua-nsi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yko",
+      "name": "Yasa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ykr",
+      "name": "Yekora",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ykt",
+      "name": "Kathu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yku",
+      "name": "Kuamasi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yky",
+      "name": "Yakoma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yla",
+      "name": "Yaul",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ylb",
+      "name": "Yaleba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yle",
+      "name": "Yele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ylg",
+      "name": "Yelogu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yli",
+      "inverted_name": "Yali, Angguruk",
+      "name": "Angguruk Yali",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yll",
+      "name": "Yil",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ylm",
+      "name": "Limi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yln",
+      "inverted_name": "Buyang, Langnian",
+      "name": "Langnian Buyang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ylo",
+      "inverted_name": "Yi, Naluo",
+      "name": "Naluo Yi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ylr",
+      "name": "Yalarnnga",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ylu",
+      "name": "Aribwaung",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yly",
+      "name": "Nyâlayu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ymb",
+      "name": "Yambes",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ymc",
+      "inverted_name": "Muji, Southern",
+      "name": "Southern Muji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ymd",
+      "name": "Muda",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yme",
+      "name": "Yameo",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ymg",
+      "name": "Yamongeri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ymh",
+      "name": "Mili",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ymi",
+      "name": "Moji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ymk",
+      "name": "Makwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yml",
+      "name": "Iamalele",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ymm",
+      "name": "Maay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ymn",
+      "name": "Yamna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ymo",
+      "name": "Yangum Mon",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ymp",
+      "name": "Yamap",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ymq",
+      "inverted_name": "Muji, Qila",
+      "name": "Qila Muji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ymr",
+      "name": "Malasar",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yms",
+      "name": "Mysian",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "ymx",
+      "inverted_name": "Muji, Northern",
+      "name": "Northern Muji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ymz",
+      "name": "Muzi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yna",
+      "name": "Aluo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ynd",
+      "name": "Yandruwandha",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yne",
+      "name": "Lang'e",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yng",
+      "name": "Yango",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ynk",
+      "inverted_name": "Yupik, Naukan",
+      "name": "Naukan Yupik",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ynl",
+      "name": "Yangulam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ynn",
+      "name": "Yana",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yno",
+      "name": "Yong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ynq",
+      "name": "Yendang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yns",
+      "name": "Yansi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ynu",
+      "name": "Yahuna",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yob",
+      "name": "Yoba",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yog",
+      "name": "Yogad",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yoi",
+      "name": "Yonaguni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yok",
+      "name": "Yokuts",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yol",
+      "name": "Yola",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yom",
+      "name": "Yombe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yon",
+      "name": "Yongkom",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "yo",
+      "alpha_3": "yor",
+      "name": "Yoruba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yot",
+      "name": "Yotti",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yox",
+      "name": "Yoron",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yoy",
+      "name": "Yoy",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ypa",
+      "name": "Phala",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ypb",
+      "inverted_name": "Phowa, Labo",
+      "name": "Labo Phowa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ypg",
+      "name": "Phola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yph",
+      "name": "Phupha",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ypm",
+      "name": "Phuma",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ypn",
+      "inverted_name": "Phowa, Ani",
+      "name": "Ani Phowa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ypo",
+      "inverted_name": "Phola, Alo",
+      "name": "Alo Phola",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ypp",
+      "name": "Phupa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ypz",
+      "name": "Phuza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yra",
+      "name": "Yerakai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yrb",
+      "name": "Yareba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yre",
+      "name": "Yaouré",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yrk",
+      "name": "Nenets",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yrl",
+      "name": "Nhengatu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yrm",
+      "name": "Yirrk-Mel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yrn",
+      "name": "Yerong",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yro",
+      "name": "Yaroamë",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yrs",
+      "name": "Yarsun",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yrw",
+      "name": "Yarawata",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yry",
+      "name": "Yarluyandi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ysc",
+      "name": "Yassic",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ysd",
+      "name": "Samatao",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ysg",
+      "name": "Sonaga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ysl",
+      "name": "Yugoslavian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ysm",
+      "name": "Myanmar Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ysn",
+      "name": "Sani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yso",
+      "name": "Nisi (China)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ysp",
+      "inverted_name": "Lolopo, Southern",
+      "name": "Southern Lolopo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ysr",
+      "inverted_name": "Yupik, Sirenik",
+      "name": "Sirenik Yupik",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yss",
+      "name": "Yessan-Mayo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ysy",
+      "name": "Sanie",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yta",
+      "name": "Talu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ytl",
+      "name": "Tanglang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ytp",
+      "name": "Thopho",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ytw",
+      "name": "Yout Wam",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yty",
+      "name": "Yatay",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yua",
+      "name": "Yucateco",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yub",
+      "name": "Yugambal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yuc",
+      "name": "Yuchi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yud",
+      "inverted_name": "Arabic, Judeo-Tripolitanian",
+      "name": "Judeo-Tripolitanian Arabic",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yue",
+      "inverted_name": "Chinese, Yue",
+      "name": "Yue Chinese",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yuf",
+      "name": "Havasupai-Walapai-Yavapai",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yug",
+      "name": "Yug",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yui",
+      "name": "Yurutí",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yuj",
+      "name": "Karkar-Yuri",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yuk",
+      "name": "Yuki",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yul",
+      "name": "Yulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yum",
+      "name": "Quechan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yun",
+      "name": "Bena (Nigeria)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yup",
+      "name": "Yukpa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yuq",
+      "name": "Yuqui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yur",
+      "name": "Yurok",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yut",
+      "name": "Yopno",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yuw",
+      "name": "Yau (Morobe Province)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yux",
+      "inverted_name": "Yukaghir, Southern",
+      "name": "Southern Yukaghir",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yuy",
+      "inverted_name": "Yugur, East",
+      "name": "East Yugur",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yuz",
+      "name": "Yuracare",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yva",
+      "name": "Yawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yvt",
+      "name": "Yavitero",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "ywa",
+      "name": "Kalou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ywg",
+      "name": "Yinhawangka",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ywl",
+      "inverted_name": "Lalu, Western",
+      "name": "Western Lalu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ywn",
+      "name": "Yawanawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ywq",
+      "inverted_name": "Yi, Wuding-Luquan",
+      "name": "Wuding-Luquan Yi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ywr",
+      "name": "Yawuru",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ywt",
+      "inverted_name": "Lalo, Xishanba",
+      "name": "Xishanba Lalo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ywu",
+      "inverted_name": "Nasu, Wumeng",
+      "name": "Wumeng Nasu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yww",
+      "name": "Yawarawarga",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yxa",
+      "name": "Mayawali",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yxg",
+      "name": "Yagara",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yxl",
+      "name": "Yardliyawarra",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yxm",
+      "name": "Yinwum",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yxu",
+      "name": "Yuyu",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yxy",
+      "name": "Yabula Yabula",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yyr",
+      "name": "Yir Yoront",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "yyu",
+      "name": "Yau (Sandaun Province)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yyz",
+      "name": "Ayizi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yzg",
+      "inverted_name": "Buyang, E'ma",
+      "name": "E'ma Buyang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "yzk",
+      "name": "Zokhuo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zaa",
+      "inverted_name": "Zapotec, Sierra de Juárez",
+      "name": "Sierra de Juárez Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zab",
+      "inverted_name": "Zapotec, Western Tlacolula Valley",
+      "name": "Western Tlacolula Valley Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zac",
+      "inverted_name": "Zapotec, Ocotlán",
+      "name": "Ocotlán Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zad",
+      "inverted_name": "Zapotec, Cajonos",
+      "name": "Cajonos Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zae",
+      "inverted_name": "Zapotec, Yareni",
+      "name": "Yareni Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zaf",
+      "inverted_name": "Zapotec, Ayoquesco",
+      "name": "Ayoquesco Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zag",
+      "name": "Zaghawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zah",
+      "name": "Zangwal",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zai",
+      "inverted_name": "Zapotec, Isthmus",
+      "name": "Isthmus Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zaj",
+      "name": "Zaramo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zak",
+      "name": "Zanaki",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zal",
+      "name": "Zauzou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zam",
+      "inverted_name": "Zapotec, Miahuatlán",
+      "name": "Miahuatlán Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zao",
+      "inverted_name": "Zapotec, Ozolotepec",
+      "name": "Ozolotepec Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zap",
+      "name": "Zapotec",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zaq",
+      "inverted_name": "Zapotec, Aloápam",
+      "name": "Aloápam Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zar",
+      "inverted_name": "Zapotec, Rincón",
+      "name": "Rincón Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zas",
+      "inverted_name": "Zapotec, Santo Domingo Albarradas",
+      "name": "Santo Domingo Albarradas Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zat",
+      "inverted_name": "Zapotec, Tabaa",
+      "name": "Tabaa Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zau",
+      "name": "Zangskari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zav",
+      "inverted_name": "Zapotec, Yatzachi",
+      "name": "Yatzachi Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zaw",
+      "inverted_name": "Zapotec, Mitla",
+      "name": "Mitla Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zax",
+      "inverted_name": "Zapotec, Xadani",
+      "name": "Xadani Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zay",
+      "name": "Zayse-Zergulla",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zaz",
+      "name": "Zari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zba",
+      "name": "Balaibalan",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "zbc",
+      "inverted_name": "Berawan, Central",
+      "name": "Central Berawan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zbe",
+      "inverted_name": "Berawan, East",
+      "name": "East Berawan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zbl",
+      "name": "Blissymbols",
+      "scope": "I",
+      "type": "C"
+    },
+    {
+      "alpha_3": "zbt",
+      "name": "Batui",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zbu",
+      "name": "Bu (Bauchi State)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zbw",
+      "inverted_name": "Berawan, West",
+      "name": "West Berawan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zca",
+      "inverted_name": "Zapotec, Coatecas Altas",
+      "name": "Coatecas Altas Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zcd",
+      "inverted_name": "Zapotec, Las Delicias",
+      "name": "Las Delicias Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zch",
+      "inverted_name": "Zhuang, Central Hongshuihe",
+      "name": "Central Hongshuihe Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zdj",
+      "inverted_name": "Comorian, Ngazidja",
+      "name": "Ngazidja Comorian",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zea",
+      "name": "Zeeuws",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zeg",
+      "name": "Zenag",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zeh",
+      "inverted_name": "Zhuang, Eastern Hongshuihe",
+      "name": "Eastern Hongshuihe Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zen",
+      "name": "Zenaga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zga",
+      "name": "Kinga",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zgb",
+      "inverted_name": "Zhuang, Guibei",
+      "name": "Guibei Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zgh",
+      "inverted_name": "Tamazight, Standard Moroccan",
+      "name": "Standard Moroccan Tamazight",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zgm",
+      "inverted_name": "Zhuang, Minz",
+      "name": "Minz Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zgn",
+      "inverted_name": "Zhuang, Guibian",
+      "name": "Guibian Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zgr",
+      "name": "Magori",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "za",
+      "alpha_3": "zha",
+      "name": "Zhuang",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zhb",
+      "name": "Zhaba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zhd",
+      "inverted_name": "Zhuang, Dai",
+      "name": "Dai Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zhi",
+      "name": "Zhire",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zhn",
+      "inverted_name": "Zhuang, Nong",
+      "name": "Nong Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "zh",
+      "alpha_3": "zho",
+      "bibliographic": "chi",
+      "name": "Chinese",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zhw",
+      "name": "Zhoa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zia",
+      "name": "Zia",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zib",
+      "name": "Zimbabwe Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zik",
+      "name": "Zimakani",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zil",
+      "name": "Zialo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zim",
+      "name": "Mesme",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zin",
+      "name": "Zinza",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ziw",
+      "name": "Zigula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ziz",
+      "name": "Zizilivakan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zka",
+      "name": "Kaimbulawa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zkb",
+      "name": "Koibal",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zkd",
+      "name": "Kadu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zkg",
+      "name": "Koguryo",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "zkh",
+      "name": "Khorezmian",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "zkk",
+      "name": "Karankawa",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zkn",
+      "name": "Kanan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zko",
+      "name": "Kott",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zkp",
+      "inverted_name": "Kaingáng, São Paulo",
+      "name": "São Paulo Kaingáng",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zkr",
+      "name": "Zakhring",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zkt",
+      "name": "Kitan",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "zku",
+      "name": "Kaurna",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zkv",
+      "name": "Krevinian",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zkz",
+      "name": "Khazar",
+      "scope": "I",
+      "type": "H"
+    },
+    {
+      "alpha_3": "zla",
+      "name": "Zula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zlj",
+      "inverted_name": "Zhuang, Liujiang",
+      "name": "Liujiang Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zlm",
+      "name": "Malay (individual language)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zln",
+      "inverted_name": "Zhuang, Lianshan",
+      "name": "Lianshan Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zlq",
+      "inverted_name": "Zhuang, Liuqian",
+      "name": "Liuqian Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zma",
+      "name": "Manda (Australia)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmb",
+      "name": "Zimba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmc",
+      "name": "Margany",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zmd",
+      "name": "Maridan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zme",
+      "name": "Mangerr",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zmf",
+      "name": "Mfinu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmg",
+      "name": "Marti Ke",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmh",
+      "name": "Makolkol",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zmi",
+      "name": "Negeri Sembilan Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmj",
+      "name": "Maridjabin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmk",
+      "name": "Mandandanyi",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zml",
+      "name": "Matngala",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zmm",
+      "name": "Marimanindji",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmn",
+      "name": "Mbangwe",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmo",
+      "name": "Molo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmp",
+      "name": "Mpuono",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmq",
+      "name": "Mituku",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmr",
+      "name": "Maranunggu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zms",
+      "name": "Mbesa",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmt",
+      "name": "Maringarr",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmu",
+      "name": "Muruwari",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zmv",
+      "name": "Mbariman-Gudhinma",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zmw",
+      "name": "Mbo (Democratic Republic of Congo)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmx",
+      "name": "Bomitaba",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmy",
+      "name": "Mariyedi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zmz",
+      "name": "Mbandja",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zna",
+      "name": "Zan Gula",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zne",
+      "name": "Zande (individual language)",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zng",
+      "name": "Mang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "znk",
+      "name": "Manangkari",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zns",
+      "name": "Mangas",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zoc",
+      "inverted_name": "Zoque, Copainalá",
+      "name": "Copainalá Zoque",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zoh",
+      "inverted_name": "Zoque, Chimalapa",
+      "name": "Chimalapa Zoque",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zom",
+      "name": "Zou",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zoo",
+      "inverted_name": "Zapotec, Asunción Mixtepec",
+      "name": "Asunción Mixtepec Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zoq",
+      "inverted_name": "Zoque, Tabasco",
+      "name": "Tabasco Zoque",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zor",
+      "inverted_name": "Zoque, Rayón",
+      "name": "Rayón Zoque",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zos",
+      "inverted_name": "Zoque, Francisco León",
+      "name": "Francisco León Zoque",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpa",
+      "inverted_name": "Zapotec, Lachiguiri",
+      "name": "Lachiguiri Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpb",
+      "inverted_name": "Zapotec, Yautepec",
+      "name": "Yautepec Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpc",
+      "inverted_name": "Zapotec, Choapan",
+      "name": "Choapan Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpd",
+      "inverted_name": "Zapotec, Southeastern Ixtlán",
+      "name": "Southeastern Ixtlán Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpe",
+      "inverted_name": "Zapotec, Petapa",
+      "name": "Petapa Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpf",
+      "inverted_name": "Zapotec, San Pedro Quiatoni",
+      "name": "San Pedro Quiatoni Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpg",
+      "inverted_name": "Zapotec, Guevea De Humboldt",
+      "name": "Guevea De Humboldt Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zph",
+      "inverted_name": "Zapotec, Totomachapan",
+      "name": "Totomachapan Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpi",
+      "inverted_name": "Zapotec, Santa María Quiegolani",
+      "name": "Santa María Quiegolani Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpj",
+      "inverted_name": "Zapotec, Quiavicuzas",
+      "name": "Quiavicuzas Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpk",
+      "inverted_name": "Zapotec, Tlacolulita",
+      "name": "Tlacolulita Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpl",
+      "inverted_name": "Zapotec, Lachixío",
+      "name": "Lachixío Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpm",
+      "inverted_name": "Zapotec, Mixtepec",
+      "name": "Mixtepec Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpn",
+      "inverted_name": "Zapotec, Santa Inés Yatzechi",
+      "name": "Santa Inés Yatzechi Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpo",
+      "inverted_name": "Zapotec, Amatlán",
+      "name": "Amatlán Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpp",
+      "inverted_name": "Zapotec, El Alto",
+      "name": "El Alto Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpq",
+      "inverted_name": "Zapotec, Zoogocho",
+      "name": "Zoogocho Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpr",
+      "inverted_name": "Zapotec, Santiago Xanica",
+      "name": "Santiago Xanica Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zps",
+      "inverted_name": "Zapotec, Coatlán",
+      "name": "Coatlán Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpt",
+      "inverted_name": "Zapotec, San Vicente Coatlán",
+      "name": "San Vicente Coatlán Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpu",
+      "inverted_name": "Zapotec, Yalálag",
+      "name": "Yalálag Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpv",
+      "inverted_name": "Zapotec, Chichicapan",
+      "name": "Chichicapan Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpw",
+      "inverted_name": "Zapotec, Zaniza",
+      "name": "Zaniza Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpx",
+      "inverted_name": "Zapotec, San Baltazar Loxicha",
+      "name": "San Baltazar Loxicha Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpy",
+      "inverted_name": "Zapotec, Mazaltepec",
+      "name": "Mazaltepec Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zpz",
+      "inverted_name": "Zapotec, Texmelucan",
+      "name": "Texmelucan Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zqe",
+      "inverted_name": "Zhuang, Qiubei",
+      "name": "Qiubei Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zra",
+      "name": "Kara (Korea)",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "zrg",
+      "name": "Mirgan",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zrn",
+      "name": "Zerenkel",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zro",
+      "name": "Záparo",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zrp",
+      "name": "Zarphatic",
+      "scope": "I",
+      "type": "E"
+    },
+    {
+      "alpha_3": "zrs",
+      "name": "Mairasi",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zsa",
+      "name": "Sarasira",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zsk",
+      "name": "Kaskean",
+      "scope": "I",
+      "type": "A"
+    },
+    {
+      "alpha_3": "zsl",
+      "name": "Zambian Sign Language",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zsm",
+      "inverted_name": "Malay, Standard",
+      "name": "Standard Malay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zsr",
+      "inverted_name": "Zapotec, Southern Rincon",
+      "name": "Southern Rincon Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zsu",
+      "name": "Sukurum",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zte",
+      "inverted_name": "Zapotec, Elotepec",
+      "name": "Elotepec Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ztg",
+      "inverted_name": "Zapotec, Xanaguía",
+      "name": "Xanaguía Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ztl",
+      "inverted_name": "Zapotec, Lapaguía-Guivini",
+      "name": "Lapaguía-Guivini Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ztm",
+      "inverted_name": "Zapotec, San Agustín Mixtepec",
+      "name": "San Agustín Mixtepec Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ztn",
+      "inverted_name": "Zapotec, Santa Catarina Albarradas",
+      "name": "Santa Catarina Albarradas Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ztp",
+      "inverted_name": "Zapotec, Loxicha",
+      "name": "Loxicha Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ztq",
+      "inverted_name": "Zapotec, Quioquitani-Quierí",
+      "name": "Quioquitani-Quierí Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zts",
+      "inverted_name": "Zapotec, Tilquiapan",
+      "name": "Tilquiapan Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ztt",
+      "inverted_name": "Zapotec, Tejalapan",
+      "name": "Tejalapan Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ztu",
+      "inverted_name": "Zapotec, Güilá",
+      "name": "Güilá Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "ztx",
+      "inverted_name": "Zapotec, Zaachila",
+      "name": "Zaachila Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zty",
+      "inverted_name": "Zapotec, Yatee",
+      "name": "Yatee Zapotec",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zua",
+      "name": "Zeem",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zuh",
+      "name": "Tokano",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_2": "zu",
+      "alpha_3": "zul",
+      "name": "Zulu",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zum",
+      "name": "Kumzari",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zun",
+      "name": "Zuni",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zuy",
+      "name": "Zumaya",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zwa",
+      "name": "Zay",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zxx",
+      "name": "No linguistic content",
+      "scope": "S",
+      "type": "S"
+    },
+    {
+      "alpha_3": "zyb",
+      "inverted_name": "Zhuang, Yongbei",
+      "name": "Yongbei Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zyg",
+      "inverted_name": "Zhuang, Yang",
+      "name": "Yang Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zyj",
+      "inverted_name": "Zhuang, Youjiang",
+      "name": "Youjiang Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zyn",
+      "inverted_name": "Zhuang, Yongnan",
+      "name": "Yongnan Zhuang",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zyp",
+      "inverted_name": "Chin, Zyphe",
+      "name": "Zyphe Chin",
+      "scope": "I",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zza",
+      "name": "Zaza",
+      "scope": "M",
+      "type": "L"
+    },
+    {
+      "alpha_3": "zzj",
+      "inverted_name": "Zhuang, Zuojiang",
+      "name": "Zuojiang Zhuang",
+      "scope": "I",
+      "type": "L"
+    }
+  ]
+}
diff --git a/lib/python3.1/site-packages/pycountry/databases/iso639-5.json b/lib/python3.1/site-packages/pycountry/databases/iso639-5.json
new file mode 100644
index 0000000000000000000000000000000000000000..edd996c857e18212d88320467ddb39782702ee4d
--- /dev/null
+++ b/lib/python3.1/site-packages/pycountry/databases/iso639-5.json
@@ -0,0 +1,464 @@
+{
+  "639-5": [
+    {
+      "alpha_3": "aav",
+      "name": "Austro-Asiatic languages"
+    },
+    {
+      "alpha_3": "afa",
+      "name": "Afro-Asiatic languages"
+    },
+    {
+      "alpha_3": "alg",
+      "name": "Algonquian languages"
+    },
+    {
+      "alpha_3": "alv",
+      "name": "Atlantic-Congo languages"
+    },
+    {
+      "alpha_3": "apa",
+      "name": "Apache languages"
+    },
+    {
+      "alpha_3": "aqa",
+      "name": "Alacalufan languages"
+    },
+    {
+      "alpha_3": "aql",
+      "name": "Algic languages"
+    },
+    {
+      "alpha_3": "art",
+      "name": "Artificial languages"
+    },
+    {
+      "alpha_3": "ath",
+      "name": "Athapascan languages"
+    },
+    {
+      "alpha_3": "auf",
+      "name": "Arauan languages"
+    },
+    {
+      "alpha_3": "aus",
+      "name": "Australian languages"
+    },
+    {
+      "alpha_3": "awd",
+      "name": "Arawakan languages"
+    },
+    {
+      "alpha_3": "azc",
+      "name": "Uto-Aztecan languages"
+    },
+    {
+      "alpha_3": "bad",
+      "name": "Banda languages"
+    },
+    {
+      "alpha_3": "bai",
+      "name": "Bamileke languages"
+    },
+    {
+      "alpha_3": "bat",
+      "name": "Baltic languages"
+    },
+    {
+      "alpha_3": "ber",
+      "name": "Berber languages"
+    },
+    {
+      "alpha_3": "bih",
+      "name": "Bihari languages"
+    },
+    {
+      "alpha_3": "bnt",
+      "name": "Bantu languages"
+    },
+    {
+      "alpha_3": "btk",
+      "name": "Batak languages"
+    },
+    {
+      "alpha_3": "cai",
+      "name": "Central American Indian languages"
+    },
+    {
+      "alpha_3": "cau",
+      "name": "Caucasian languages"
+    },
+    {
+      "alpha_3": "cba",
+      "name": "Chibchan languages"
+    },
+    {
+      "alpha_3": "ccn",
+      "name": "North Caucasian languages"
+    },
+    {
+      "alpha_3": "ccs",
+      "name": "South Caucasian languages"
+    },
+    {
+      "alpha_3": "cdc",
+      "name": "Chadic languages"
+    },
+    {
+      "alpha_3": "cdd",
+      "name": "Caddoan languages"
+    },
+    {
+      "alpha_3": "cel",
+      "name": "Celtic languages"
+    },
+    {
+      "alpha_3": "cmc",
+      "name": "Chamic languages"
+    },
+    {
+      "alpha_3": "cpe",
+      "name": "Creoles and pidgins, English‑based"
+    },
+    {
+      "alpha_3": "cpf",
+      "name": "Creoles and pidgins, French‑based"
+    },
+    {
+      "alpha_3": "cpp",
+      "name": "Creoles and pidgins, Portuguese-based"
+    },
+    {
+      "alpha_3": "crp",
+      "name": "Creoles and pidgins"
+    },
+    {
+      "alpha_3": "csu",
+      "name": "Central Sudanic languages"
+    },
+    {
+      "alpha_3": "cus",
+      "name": "Cushitic languages"
+    },
+    {
+      "alpha_3": "day",
+      "name": "Land Dayak languages"
+    },
+    {
+      "alpha_3": "dmn",
+      "name": "Mande languages"
+    },
+    {
+      "alpha_3": "dra",
+      "name": "Dravidian languages"
+    },
+    {
+      "alpha_3": "egx",
+      "name": "Egyptian languages"
+    },
+    {
+      "alpha_3": "esx",
+      "name": "Eskimo-Aleut languages"
+    },
+    {
+      "alpha_3": "euq",
+      "name": "Basque (family)"
+    },
+    {
+      "alpha_3": "fiu",
+      "name": "Finno-Ugrian languages"
+    },
+    {
+      "alpha_3": "fox",
+      "name": "Formosan languages"
+    },
+    {
+      "alpha_3": "gem",
+      "name": "Germanic languages"
+    },
+    {
+      "alpha_3": "gme",
+      "name": "East Germanic languages"
+    },
+    {
+      "alpha_3": "gmq",
+      "name": "North Germanic languages"
+    },
+    {
+      "alpha_3": "gmw",
+      "name": "West Germanic languages"
+    },
+    {
+      "alpha_3": "grk",
+      "name": "Greek languages"
+    },
+    {
+      "alpha_3": "hmx",
+      "name": "Hmong-Mien languages"
+    },
+    {
+      "alpha_3": "hok",
+      "name": "Hokan languages"
+    },
+    {
+      "alpha_3": "hyx",
+      "name": "Armenian (family)"
+    },
+    {
+      "alpha_3": "iir",
+      "name": "Indo-Iranian languages"
+    },
+    {
+      "alpha_3": "ijo",
+      "name": "Ijo languages"
+    },
+    {
+      "alpha_3": "inc",
+      "name": "Indic languages"
+    },
+    {
+      "alpha_3": "ine",
+      "name": "Indo-European languages"
+    },
+    {
+      "alpha_3": "ira",
+      "name": "Iranian languages"
+    },
+    {
+      "alpha_3": "iro",
+      "name": "Iroquoian languages"
+    },
+    {
+      "alpha_3": "itc",
+      "name": "Italic languages"
+    },
+    {
+      "alpha_3": "jpx",
+      "name": "Japanese (family)"
+    },
+    {
+      "alpha_3": "kar",
+      "name": "Karen languages"
+    },
+    {
+      "alpha_3": "kdo",
+      "name": "Kordofanian languages"
+    },
+    {
+      "alpha_3": "khi",
+      "name": "Khoisan languages"
+    },
+    {
+      "alpha_3": "kro",
+      "name": "Kru languages"
+    },
+    {
+      "alpha_3": "map",
+      "name": "Austronesian languages"
+    },
+    {
+      "alpha_3": "mkh",
+      "name": "Mon-Khmer languages"
+    },
+    {
+      "alpha_3": "mno",
+      "name": "Manobo languages"
+    },
+    {
+      "alpha_3": "mun",
+      "name": "Munda languages"
+    },
+    {
+      "alpha_3": "myn",
+      "name": "Mayan languages"
+    },
+    {
+      "alpha_3": "nah",
+      "name": "Nahuatl languages"
+    },
+    {
+      "alpha_3": "nai",
+      "name": "North American Indian languages"
+    },
+    {
+      "alpha_3": "ngf",
+      "name": "Trans-New Guinea languages"
+    },
+    {
+      "alpha_3": "nic",
+      "name": "Niger-Kordofanian languages"
+    },
+    {
+      "alpha_3": "nub",
+      "name": "Nubian languages"
+    },
+    {
+      "alpha_3": "omq",
+      "name": "Oto-Manguean languages"
+    },
+    {
+      "alpha_3": "omv",
+      "name": "Omotic languages"
+    },
+    {
+      "alpha_3": "oto",
+      "name": "Otomian languages"
+    },
+    {
+      "alpha_3": "paa",
+      "name": "Papuan languages"
+    },
+    {
+      "alpha_3": "phi",
+      "name": "Philippine languages"
+    },
+    {
+      "alpha_3": "plf",
+      "name": "Central Malayo-Polynesian languages"
+    },
+    {
+      "alpha_3": "poz",
+      "name": "Malayo-Polynesian languages"
+    },
+    {
+      "alpha_3": "pqe",
+      "name": "Eastern Malayo-Polynesian languages"
+    },
+    {
+      "alpha_3": "pqw",
+      "name": "Western Malayo-Polynesian languages"
+    },
+    {
+      "alpha_3": "pra",
+      "name": "Prakrit languages"
+    },
+    {
+      "alpha_3": "qwe",
+      "name": "Quechuan (family)"
+    },
+    {
+      "alpha_3": "roa",
+      "name": "Romance languages"
+    },
+    {
+      "alpha_3": "sai",
+      "name": "South American Indian languages"
+    },
+    {
+      "alpha_3": "sal",
+      "name": "Salishan languages"
+    },
+    {
+      "alpha_3": "sdv",
+      "name": "Eastern Sudanic languages"
+    },
+    {
+      "alpha_3": "sem",
+      "name": "Semitic languages"
+    },
+    {
+      "alpha_3": "sgn",
+      "name": "sign languages"
+    },
+    {
+      "alpha_3": "sio",
+      "name": "Siouan languages"
+    },
+    {
+      "alpha_3": "sit",
+      "name": "Sino-Tibetan languages"
+    },
+    {
+      "alpha_3": "sla",
+      "name": "Slavic languages"
+    },
+    {
+      "alpha_3": "smi",
+      "name": "Sami languages"
+    },
+    {
+      "alpha_3": "son",
+      "name": "Songhai languages"
+    },
+    {
+      "alpha_3": "sqj",
+      "name": "Albanian languages"
+    },
+    {
+      "alpha_3": "ssa",
+      "name": "Nilo-Saharan languages"
+    },
+    {
+      "alpha_3": "syd",
+      "name": "Samoyedic languages"
+    },
+    {
+      "alpha_3": "tai",
+      "name": "Tai languages"
+    },
+    {
+      "alpha_3": "tbq",
+      "name": "Tibeto-Burman languages"
+    },
+    {
+      "alpha_3": "trk",
+      "name": "Turkic languages"
+    },
+    {
+      "alpha_3": "tup",
+      "name": "Tupi languages"
+    },
+    {
+      "alpha_3": "tut",
+      "name": "Altaic languages"
+    },
+    {
+      "alpha_3": "tuw",
+      "name": "Tungus languages"
+    },
+    {
+      "alpha_3": "urj",
+      "name": "Uralic languages"
+    },
+    {
+      "alpha_3": "wak",
+      "name": "Wakashan languages"
+    },
+    {
+      "alpha_3": "wen",
+      "name": "Sorbian languages"
+    },
+    {
+      "alpha_3": "xgn",
+      "name": "Mongolian languages"
+    },
+    {
+      "alpha_3": "xnd",
+      "name": "Na-Dene languages"
+    },
+    {
+      "alpha_3": "ypk",
+      "name": "Yupik languages"
+    },
+    {
+      "alpha_3": "zhx",
+      "name": "Chinese (family)"
+    },
+    {
+      "alpha_3": "zle",
+      "name": "East Slavic languages"
+    },
+    {
+      "alpha_3": "zls",
+      "name": "South Slavic languages"
+    },
+    {
+      "alpha_3": "zlw",
+      "name": "West Slavic languages"
+    },
+    {
+      "alpha_3": "znd",
+      "name": "Zande languages"
+    }
+  ]
+}
diff --git a/lib/python3.1/site-packages/pycountry/db.py b/lib/python3.1/site-packages/pycountry/db.py
new file mode 100644
index 0000000000000000000000000000000000000000..08f24e8c97b8329de73108957def0a2b2275ec6c
--- /dev/null
+++ b/lib/python3.1/site-packages/pycountry/db.py
@@ -0,0 +1,200 @@
+import json
+import logging
+import threading
+from typing import Any, Iterator, List, Optional, Type, Union
+
+logger = logging.getLogger("pycountry.db")
+
+
+class Data:
+    def __init__(self, **fields: str):
+        self._fields = fields
+
+    def __getattr__(self, key):
+        if key in self._fields:
+            return self._fields[key]
+        raise AttributeError(key)
+
+    def __setattr__(self, key: str, value: str) -> None:
+        if key != "_fields":
+            self._fields[key] = value
+        super().__setattr__(key, value)
+
+    def __repr__(self) -> str:
+        cls_name = self.__class__.__name__
+        fields = ", ".join("%s=%r" % i for i in sorted(self._fields.items()))
+        return f"{cls_name}({fields})"
+
+    def __dir__(self) -> List[str]:
+        return dir(self.__class__) + list(self._fields)
+
+    def __iter__(self):
+        # allow casting into a dict
+        for field in self._fields:
+            yield field, getattr(self, field)
+
+
+class Country(Data):
+    pass
+
+
+class Subdivision(Data):
+    pass
+
+
+def lazy_load(f):
+    def load_if_needed(self, *args, **kw):
+        if not self._is_loaded:
+            with self._load_lock:
+                self._load()
+        return f(self, *args, **kw)
+
+    return load_if_needed
+
+
+class Database:
+    data_class: Union[Type, str]
+    root_key: Optional[str] = None
+    no_index: List[str] = []
+
+    def __init__(self, filename: str) -> None:
+        self.filename = filename
+        self._is_loaded = False
+        self._load_lock = threading.Lock()
+
+        if isinstance(self.data_class, str):
+            self.factory = type(self.data_class, (Data,), {})
+        else:
+            self.factory = self.data_class
+
+    def _clear(self):
+        self._is_loaded = False
+        self.objects = []
+        self.index_names = set()
+        self.indices = {}
+
+    def _load(self) -> None:
+        if self._is_loaded:
+            # Help keeping the _load_if_needed code easier
+            # to read.
+            return
+        self._clear()
+
+        with open(self.filename, encoding="utf-8") as f:
+            tree = json.load(f)
+
+        for entry in tree[self.root_key]:
+            obj = self.factory(**entry)
+            self.objects.append(obj)
+            # Inject into index.
+            for key, value in entry.items():
+                if key in self.no_index:
+                    continue
+                # Lookups and searches are case insensitive. Normalize
+                # here.
+                index = self.indices.setdefault(key, {})
+                value = value.lower()
+                if value in index:
+                    logger.debug(
+                        "%s %r already taken in index %r and will be "
+                        "ignored. This is an error in the databases."
+                        % (self.factory.__name__, value, key)
+                    )
+                index[value] = obj
+
+        self._is_loaded = True
+
+    # Public API
+
+    @lazy_load
+    def add_entry(self, **kw):
+        # create the object with the correct dynamic type
+        obj = self.factory(**kw)
+
+        # append object
+        self.objects.append(obj)
+
+        # update indices
+        for key, value in kw.items():
+            if key in self.no_index:
+                continue
+            value = value.lower()
+            index = self.indices.setdefault(key, {})
+            index[value] = obj
+
+    @lazy_load
+    def remove_entry(self, **kw):
+        # make sure that we receive None if no entry found
+        if "default" in kw:
+            del kw["default"]
+        obj = self.get(**kw)
+        if not obj:
+            raise KeyError(
+                f"{self.factory.__name__} not found and cannot be removed: {kw}"
+            )
+
+        # remove object
+        self.objects.remove(obj)
+
+        # update indices
+        for key, value in obj:
+            if key in self.no_index:
+                continue
+            value = value.lower()
+            index = self.indices.setdefault(key, {})
+            if value in index:
+                del index[value]
+
+    @lazy_load
+    def __iter__(self) -> Iterator["Database"]:
+        return iter(self.objects)
+
+    @lazy_load
+    def __len__(self) -> int:
+        return len(self.objects)
+
+    @lazy_load
+    def get(
+        self, *, default: Optional[Any] = None, **kw: Optional[str]
+    ) -> Optional[Any]:
+        if len(kw) != 1:
+            raise TypeError("Only one criteria may be given")
+        field, value = kw.popitem()
+        if not isinstance(value, str):
+            raise LookupError()
+        # Normalize for case-insensitivity
+        value = value.lower()
+        index = self.indices[field]
+        try:
+            return index[value]
+        except KeyError:
+            # Pythonic APIs implementing     get() shouldn't raise KeyErrors.
+            # Those are a bit unexpected and they should rather support
+            # returning `None` by default and allow customization.
+            return default
+
+    @lazy_load
+    def lookup(self, value: str) -> Type:
+        if not isinstance(value, str):
+            raise LookupError()
+
+        # Normalize for case-insensitivity
+        value = value.lower()
+
+        # Use indexes first
+        for key in self.indices:
+            try:
+                return self.indices[key][value]
+            except LookupError:
+                pass
+
+        # Use non-indexed values now. Avoid going through indexed values.
+        for candidate in self:
+            for k in self.no_index:
+                v = candidate._fields.get(k)
+                if v is None:
+                    continue
+                if v.lower() == value:
+                    return candidate
+
+        raise LookupError("Could not find a record for %r" % value)
diff --git a/lib/python3.1/site-packages/pycountry/locales/ab/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ab/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..09f6341ab878de657eae07bf810f2539820b7187
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ab/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ab/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/ab/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..2193d1d3fe5d5cce95c39add6652756ab4f08332
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ab/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ace/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ace/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3294922fd469fb706780151a782e3b1e5aacb132
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ace/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ach/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ach/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..27643b8debab487df43b72bfc831e658ad7f97e5
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ach/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/af/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/af/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4044ce811426842da9978bfced16eae242e4de99
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/af/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/af/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/af/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ba97f187ddf42b6f90c35f0fea336431c3d1b1ab
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/af/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/af/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/af/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..667f28beb748e7796388149c04477ab805296363
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/af/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ak/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ak/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4277660655fdf4fc2844c7a326f86daae82c3229
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ak/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/am/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/am/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4c3f214fda5ef900e93ef39c1556880dd03d6e4d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/am/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/am/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/am/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..b009b1d468a721b48e3c6b67344862336437607b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/am/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/am/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/am/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8d03b467f0da7389b8de6f33c134966febeb5d23
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/am/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/an/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/an/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..42d853559b63a05a72b86db872c4b8c0d3ae67ba
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/an/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..b09763ee9499535219101544620d8390fefbe6db
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..cd863f5c12ebb4d02b5eecd37533eaca2e7fd95c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..507e762958aca9ffff5f860a2a40b58848e0d594
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..663726c43c8cc3383cbaac785917c458a295424f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8749de698706cf492489d56043016c2654302ce9
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ar/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/as/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/as/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e30b1354ec03a62fea6755e8ff8fcb6e33d7d966
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/as/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/as/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/as/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3a85d8670baf433fc1526cbffccf57ff2229f0bc
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/as/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1510c57a87a42a2520ff81ac45945dda303c70ac
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c42640f8c0943b5889384d7b196e9c550a8aeb49
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d3c3d7e1e6fac10c7e43c6403d262a6748ad8f3b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..6e1a10431ceffcb394d500484eb2acfd3921fcd6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..76e3883c71f4313b028a8457b8a77c54a71bad9c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4708f32917cc0a911c765169089edff0d52a1068
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ast/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ay/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ay/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..722bb6dec3d6ed55e4c6292a611637d0e6312508
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ay/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/az/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/az/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fd2cd1e3a8aa3b7db485647e5b8fc7de90e1f37f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/az/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/az/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/az/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fb29b8628752cef710a14e93d2bcffa9fe97f2c0
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/az/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/az/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/az/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8dc5cc7005e9fd16b6cb3a385151e8c14d6e1970
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/az/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/az/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/az/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4ff1517bd1f646ad80b5721a4571e2b8fd954c90
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/az/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ba/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ba/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fed8c412ad86e4f1055cac69884a6576b9105d99
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ba/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bar/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/bar/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1f6607e5272706317e954ab57b23ee45b4e41e34
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bar/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..02e613710740c9b28545c718906804f5dcc4c02d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1b002b540828157eabcab61fa3170c83fd2fb974
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..dcb9842540d86218586de61906c171da8dd9ff37
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..168318d5e5648b4aee6d8165fba5e894735d8f1a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f69e8c204a0d4ff81cbf621ceb2153d8cb60011f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..503d8e95ffda18fb669d61cfda355754cf4bc5c1
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/be/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..dbfdb6ffe8f2a8a6b6311d1aeb594753add1bf97
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e3be16e56ea6445746bf1e4555d2567709153b6a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..958264c18336ada640031728556058f633e5e6bd
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0d3037eabc8825ee430412002bd52bc25aef18c4
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0c92dc9c4323b574e0ff98c050d645a577f303ab
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bg/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bi/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/bi/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3ca1db7a319b2fd643c1828dc571cf33c86207ca
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bi/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fb134cc14fd04e8b1f5e15c14d45f5d0ba796288
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f5e17012f2772a9055b2f1491ccf67d9d949033a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d6c0f08fbd7d9545389a32c7e159fb8cfb7adec6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..17012ef782e38dd902444afedb72d9adea67fdd6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..87024da6c43ceccedbe3f3db6e1813b1d8e1dcf2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bn/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bn_BD/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/bn_BD/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..bee7a566bf76568ed8b54c4da600dd8aff99de2d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bn_BD/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bn_BD/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/bn_BD/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e243d57d52628aca576f42f7306223433de3472c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bn_BD/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bn_BD/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/bn_BD/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4d13bb310405eeb72a43dbfe0cb949c468ce3b71
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bn_BD/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bn_IN/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/bn_IN/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f6ac138f7f5345ab13d1d8ca594f56d6f2daa2c1
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bn_IN/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bn_IN/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/bn_IN/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..83fa1aa641e4b23d17ae3fe9ff29f80d31ff8af3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bn_IN/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..46130822fff0866a1cd023b0b7d3378cc08b2525
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1f793fb525baa8532811bae0db71a29422a42fdd
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..46fe2d87e2615ba40102dfd9ca8c6a02f4428566
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4ba7f93a327c17a837b238ccc07d279235b5d8a4
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a194e2e8d7c4f8c4fd2a15c2f70bce3b2c3ae4e7
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/br/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bs/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/bs/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..9f95ff3d66383e2ee2dcf307a9349b8f3bb43db4
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bs/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bs/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/bs/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..bce0d0c567a60b1822f535efee6f247cc25e55dc
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bs/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bs/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/bs/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d4de4daada3054f654418b82d6916d48619983a8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bs/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/bs/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/bs/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..393cd4a3b61dab98e1ea7ee55d627941ead72f00
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/bs/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/byn/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/byn/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a31f539e03f1d147b5fc44141b88cab89c20a4de
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/byn/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/byn/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/byn/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..6aa6db6f45fceae9120357032c2c8a016aa39420
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/byn/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/byn/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/byn/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c456f77362ad54643f85273e573d8ef2fc80b101
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/byn/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a06cc55429e8084ac2a53bcb197cae9ea42aa3bb
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..9b00915c291cb8507ce091327446df4c8f828c71
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8416b0cbe72b6b7012d90294b7c5aa37799c1fb2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..13beeb80ebc77f0fd05ea553b8ea00b8ee87afa8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f57c5f6dcdd3e6c8267ce853d1779bb11c070ff5
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4db8b6bf133cd91015f30074beb48b051e7a29c3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ca/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ce/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ce/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c1646fc1a7edba250112748d74627f183aa4386f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ce/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ch/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ch/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..079e2d72e0c14f2b12ac38cb337899eabb9aee43
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ch/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/chr/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/chr/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a3778663c3e797261683d761b45025d40b0f8efd
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/chr/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ckb/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ckb/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..11b3a5e331fc567152d1eea8d20ec5cbbaedc33b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ckb/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/crh/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/crh/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a427bf2f2c1895598f7f3df5d0e460bf7f875c4b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/crh/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/crh/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/crh/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..b2951254e128c2be4109f1a95ecd08152f416ebc
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/crh/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/crh/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/crh/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ba9694524221b4789c2c9a016a1d82e9291e9d64
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/crh/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..2494329057ab4adc1ed3e2783807f5e7f53d4271
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..61ec42a33cb1cfa2a00fc6d9ee6e7d02dbb4d110
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7810c1f03277416e16c228947054fa38aa6655d4
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e263b7388abc680d2091e4ce623ed2a8647322b7
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0dfefee97429d743fd3863550cea3afc811b0c11
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..864f3d6357121aa04c3b95cbc85955d851beb297
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..134e17b44c8a66ea9bf1f72ce6e7e687662d04d3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cs/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/csb/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/csb/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..66ce3691037df606fb0a53ba1a7ca6f47c5603f0
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/csb/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cv/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/cv/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..733492fb3ccffaf9e50076a3dfbc8f4b59748a55
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cv/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cv/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/cv/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..31e566f4a0d50bf0244c0a1a48294590ec794cc0
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cv/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..871bd59815bd5ba8563885b04dfc892af276d263
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..56de878d050390ebbe818e378cbec38f989373ad
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7cd1e36f24724b50b98be0adbfd1f42b3524c89b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..80a8bf1dd28e1581b1cab20a3144abdfe49ced55
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d3cee86d55e442d6c432dd8d9d94dc33fd361f11
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..06f627d2d4a687dd9d82b7e1c4b13ac951e384c3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..563106c247caed0dbe5731aea841fb9f29a02545
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/cy/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f1d545552b13585a9aafbd2224ab1a311221c020
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..64e8987658e4809d2d9e338743e4a69cd51f2430
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fa9c504854e038f4e62adf5df05231a765b17413
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..49775aadf9e0b2857f7c917a4f5179630dc8d308
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e8987d6bfd102fc0e6fb958e84ef706c4b9f9e71
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1a0e9ac6ea5501090daf37868a1ea530d63d195d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/da/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..aa57692104823ffcaaa774547c335f21595a128b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..41b1f7e8eee1e96f193ee8f0ffa6e4bf4257b98d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3121abff5a56c9827a47366646794705661f539c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..02594130b08f8584e8ee846bae72b213c9656610
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..89d9b1c39023810090edca5dc90fddd81b41d3cf
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/de/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/dv/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/dv/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0a0f8de252b0356775b8ba88285f3dac58aaf57b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/dv/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/dz/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/dz/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1680ba6fa82aa68de9c8a62142b9b63000568318
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/dz/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/dz/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/dz/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0603b3caef79446db0f5dfb98b12936a694794fa
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/dz/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ee/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ee/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..5ffd23198a8d17425487305914799feff60c5aeb
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ee/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..b24d46cb36afd0a21f7524bfa571dc225e27f0e5
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7dd6d789388144dbf33b09d4fd0280fc986a0a46
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a4274f7369b5874d2b0e5edc3a9cf03b7b844ddb
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..b0a65c7f8bddada39fae1e4e233381e42f82e265
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c08a168e6dd9d1e64991a1a2eafe82576f6910b9
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..591d3ff7f4390a17e5de65c19e420211d3ec7517
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7eea0aa3ee79bd806a27ea6c1e22f26d87c63d8e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/el/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/en/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/en/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c8a4213b10a7b68dce0d521a45581694dd6a856a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/en/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..04c35cd16cfa817c5e19e04ca8cda80e5f4f62ce
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..646542ad99ad9f8aa39625e26a7506010e1352ca
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..2a3351dd5c6920f187a86acca89b3edfd0b9b994
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7a95bdc53396da26dd1e2e6414a90f6d549b3171
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..b2fdb3735865f59814e2340dcc64253a8d187e4c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..151d60f0db41ba040f8cafb4e32da99db54ca936
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..566c6a261794d0939891ac0538facf5d472b8d58
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/eo/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..22f1649275f007940502b7db71d1913d61d6f226
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..6a8cbefa6f3c627161a47a0082da1fee54780898
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..02967c2d3f41f05de0a560f70dbebda45f2cab48
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..38b5758d901be51c30ef5de857b67dada65bc337
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..94e820e5b7c38865dbcb7373f97fd8aa144ae83a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..58da57013fa380fccea14d76eddb2ec19f5d88dd
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/es/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..b91269c6ffbe0114efa55e239314fb9e2195d793
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..46b473d13462650ecb95cbb319f2a81e2031e32d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a91e2cf3dccb9618ac57cf4514f65dfe1e4070ec
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d618e9d9560db57c2b4e38116e1bfa26b20e7e3d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d30ce668386c691415e26bf8c50622695b700f41
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ccf9c094ba4d6bc99acfd2d2fa83c2246045f5c2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..6b36fc1cda982bce4caa4e4656bc42365ccb9120
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/et/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..bbb7cca31ff990358084b951e0f09e5672c6a401
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d974098fb67261fd4e6ecf6cce001ce71ad41df8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..301a8bc11b2a094575e827c316f163ba2d542987
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fee829ea1a6c15fc6120a2019ec78bc1b5b4dce3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..59a873c92966a967d02498ca786d05940fa315b1
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d3395e7470972a3cd67ca439c7dcb3c5811d648d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/eu/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e9752e26119ad8fa5f379573def99aefdb7aff04
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f542be84e0c2247f6370a0cc9acbd0bfc6afc6fa
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..095f574587b5d71aa02736bc928c6d326d9d4ace
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4cbe061971ff8d73c036ca290fbc3d9f45674315
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..223f2852d7d9ab64e9506dd2466ab045f7436ff1
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fa/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ff/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ff/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d02090d4039ccb1b7bc2440a3dcbf284328b89c7
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ff/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c5df0cc99e43249eaacf5fcbc7270ab051f4da81
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..370127273971bff8c6ca0210f21fa687912e9312
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c4444d1abb660190b9e241d058a93ef2f8071d8c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0d0164e2a805f78f9df9c54aea57bdbdb91e9686
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d77bb3586cd4fecabe119e3fbbc547faac5655fe
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a097d20a7fd44dd33203276d82ca377fe319e64e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fi/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fil/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/fil/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e8e857c3a9692559dc6b3d8da8d57ab211142979
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fil/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fil/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/fil/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d5602bc067774d1304176b72d3e39f43a9392f6b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fil/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fil/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/fil/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..35b5fde0b192a5cf2af26fea8260f50331295b34
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fil/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fo/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/fo/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..23327ae2dd4f6a60dbab88faee267f8bbfcbcebf
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fo/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fo/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/fo/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4e36f12c00c2d51765a027514c88ae265e42fefa
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fo/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..62b78b1843917c3b19924cfcff24cf15e1a8ee54
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..65516dd1c37a5b9bcb712bf73a725a5a9626f441
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3e76d0c1f83c6d7f7e2526905e0d5605f6c6a20a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a73aa850da75e63bcc67221bf650602df276cbb8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3e6f04de0626d0b861ced5bf639b78773c0c4cfb
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fr/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/frp/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/frp/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c261a3a4c29fac0e2553e7065d49aa8460ee20ae
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/frp/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fur/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/fur/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ca89847bbf801be43435b4a106bb98b6fab6f020
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fur/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fur/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/fur/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7e3ac265ff63e56c406505b24a67f4d65dbbe8cd
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fur/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fur/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/fur/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..56d6c4077f0bb87a8de8ffbf03912a47923226a2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fur/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fur/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/fur/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..6323898b10c4fc23fa1849ed9ccf8321427f3e30
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fur/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/fy/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/fy/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fc326954deab730f9585a4c99ead816ec307eedd
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/fy/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..529b111dff987535d550bb64071ee519718a008a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e4e5029443e361e45a0a1d4312df40ec52d2892e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0a9dd28cd0a26ecb5a496ff8690c3a7926fe7bee
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a8a49df94e346c4627c7309d35f84cc1b6208a3b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c5611168ebfbaca9517d776cda635a923a06d57d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ga/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/gez/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/gez/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7fcd8057c80ab9c340d1e7fcb1cac21dfc5cea3b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/gez/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/gez/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/gez/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..aff223a46d966757f99490d8f861261121f4eb8d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/gez/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/gez/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/gez/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..636b0124eed81c50eb0a2f83a56e42a42c6b7e22
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/gez/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/gl/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/gl/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..05e14408478191c22288cc5e955c07fccde66198
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/gl/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/gl/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/gl/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d123d3959fd8ee20af2a424ff9c90003a915c0bd
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/gl/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/gl/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/gl/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c28f802b17ee3fe6d606c5eb95c9ca01e8ce87b6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/gl/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/gl/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/gl/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..56657f816f80322e69b80e533f1f64459003448d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/gl/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/gn/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/gn/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..895241bb2cfc516f9273d0b8d1440fe8dc444def
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/gn/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/gu/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/gu/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c8e05c3074768ca6630e120d1ed9139649526f5f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/gu/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/gu/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/gu/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8563fa252c29e3a4800ad903b67ee1852a4ccaab
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/gu/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/gu/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/gu/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ba1d2c07e8688b3547602165f27e34fe69c9e7a2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/gu/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/gv/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/gv/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..251f9ec4cae72933df9ed749d241aba970ab0982
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/gv/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ha/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ha/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..eab9445202cbccccb20fba20a07c42a040a54acb
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ha/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/haw/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/haw/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..2a78df8e01ee75af9d187b9351d02bedd10f5d76
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/haw/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/haw/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/haw/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..525110273ea4e7850319e9bd0b81a1b2c2041f84
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/haw/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f91a8207e7b98380744ce9c941bdf5d78ec27c39
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..5fd134ecda45b0a2f4e4ad8e2c7b36bf87bd2b3f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c16da1986af0d07c59b63084bce726d82b794f3e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..cc25f2a4d634e41f7b62af0ade868f80d24e8b89
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..140986ab9fab7f51935550e42733b2ce5b4d6314
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/he/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hi/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/hi/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c1b737970d43dfee134cf1b93d29ddf63ca868c8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hi/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hi/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/hi/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..6b8879abc2aaa2137484c2b34b7518f7e2c7c5c4
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hi/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hi/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/hi/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..32fab94a8b09b1aac3ef9fe1becf5e0bdea80a6d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hi/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hi/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/hi/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ddb95fcd4dfdfb4e3edc2c93a15947a09b7e8fd3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hi/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..46d4653416a4dc01467ad2b00ce153c89045b8e2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..028574d9d2677822bed4bc87d33dc518c9bc15a3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ab220d1bb55e6813b3b77af20d2c2fd07ca00a82
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3c621207f679c7b4ab500914988f9c9ae71af6b7
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..9dbfae6b4495ece025cfd56347dd738eb8459951
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..73b6468a5ed4a2545c347f82a82aa43f2607c55e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ef6a83ff0ff93376a08290e58ae2b2509af5af56
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hr/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ht/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ht/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8bb05ab18f51c1dcb9bac59e119dbb4e29993f0c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ht/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7de88e89acd172eb6fe9f55c92abc7ac9cba956a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..36a7cf92315deab7f4cac2c7b3135bb39379a25a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..b1b580d5627a3fe55112aa776aa2c0efe5362c3e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..5c5c60ce6d8cc80a5e806ca9b708cbd979d31418
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0da2c7d5d2e351624f886d91bfb9d840705cec18
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f50f62568bfc799ae338023714d2d9978f197779
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1d8ef446d9a90a5f6cdc13a3cf34ef2727d5cab3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hu/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hy/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/hy/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7805d35f329a0ef84f1eddf1a9aea7574f31deb2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hy/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/hy/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/hy/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8f52c3d373c4e883b0c72dc813131d1c216fdf46
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/hy/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ia/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/ia/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..64396b5063c6931a3c74d4cb53faebe142af3984
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ia/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ia/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ia/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..2c051683b84ea8fde7b9bc1b380872479160f05c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ia/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ia/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ia/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..85888c8b9b8749edae8da6622113a3612c79c935
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ia/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fa262dbde52ea537733111e7c971a72610bbfd03
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a43f00f2bdf863e23bde58ffb11a523138273a5f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..be1350233007a3e89837414cad8f59298893cfe7
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..084bf15c835b57bca21e999ac2f3fb77df371fea
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1774f29a64cc52de4e0c652f259eb731d9dfab81
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/id/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/io/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/io/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c59bd40d9a678b797fa1dacf0703c19034967da6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/io/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..49c4ceccc3125207730e3f4a204eda7f0b822a94
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..910933f63c468dcb08902d40fac51cb8f6e9a5b1
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0b4b2b28a5c339a35d513f6746a8a1acea201e21
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..26dff1856a9e39faa44e1e5b63d24878d7458c62
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f8e25cd33669de5a8a512c864480fdc5bcb975fa
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..29bc118b124c3ee812567b6cf4196e86dd30f52b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/is/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f8c5f967bf65868675ecb92e7e4c78d7149313e8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c85f044b6c218cc85917faf0ae004cce29336b1d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e8bbe155053941f6766a105d0b11db0a46b0ae16
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0676d459890ac0cf3b38e8d9d80a4462b05bea33
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..90e46bcb881e20036eaa500ec52ee39bd197c679
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/it/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/iu/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/iu/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8a99875164ccab3cc7b9760ce4f5b6b9acc26d64
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/iu/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c547a64241116ef6b3d73e906f91334c40cb5995
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e4501e1ba0df845e85081a3f354abd700394e25b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4d483138edb67a2c157db5562b89486b5ea7490f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..6018191b44861744bff3a717f1c1322016cc8647
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e69f69a4c4f06760698c2cd888666b4f1474d78c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..2ac10ba11526e4c94e6d0ceac3fed73bdbcb9288
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ja/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/jam/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/jam/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..59023fef14e6900134bc2133e9ab3a3bacc80f5a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/jam/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..124d5a494197a8bb2c744b351b94ec8a2372e884
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..454b479fb21d7269a4b1c863e6568fe777d0874f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..45013e1900705641ede08f6692872c75623ff1ae
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..40d0e71263e56ba4acf814d04773824b4a575735
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..215739ee159c0a728f86b0a800d48a33f6c089c5
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d05170a17ce5a6af2be492acdaf12cb5d51a8311
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ka/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..5ee7f7a32f8ccc5cd3444309e47deeb696734fce
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7a34ec436cbb2b1cf5cbea69c95914f19ac73b0f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3c7467b120aea039acafac8eef45247d2fb273f3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..5bdf5f3b509d291b50e7ad641a768f5e2b3e6780
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..04d79b0cb7f62b020f1a7b0e0c5c6921bd71b889
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1d570453dd8cb3b88c3e03da456ad96cee153ca9
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kab/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ki/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ki/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..eda5b272bea20c8234f08ed85ba12004d47d38a3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ki/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kk/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/kk/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e038a15824cd762c857600d913edb9f4e89b26d4
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kk/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kk/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/kk/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e1ebf06a95cec55e03127532dd86e5ebe846812c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kk/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kl/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/kl/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a7c5b2c0f8305e1d5ef442bd2bb302280c904abd
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kl/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/km/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/km/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..476ae01774b70fdfd304c997ba72f034fc26f145
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/km/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/km/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/km/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0bd63b419c22941e4500fb430fc975e71c4ea240
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/km/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/km/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/km/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c5d88e51e3f693e08a6948b367dd725d7186c970
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/km/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/km/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/km/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e152336c14564548db878ceb08ca631b7271554b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/km/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kmr/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/kmr/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ab691df91ba37959e6ab4ca41775e15d6bce6f3f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kmr/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kmr/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/kmr/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4ea4ca4808db62f112857cf5d2b1b28a66a43f5a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kmr/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kmr/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/kmr/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..9fb6b6679d2bbe1438b5c05eb056364e927107ef
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kmr/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kn/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/kn/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..042784a54e294d048bcdf19c92b5f32bf79e5737
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kn/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kn/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/kn/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..afbace45213be66e407de2fa36cc51b54865efb1
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kn/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7fb82448ec44ef718ad71feaf61c70a23a8eb7d9
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..175b07183850d577b35aba6ac7e1a5066bd67a01
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c9b613abc0ccf89c7187bc8b7631f07360a489d2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e36ab74d8c5eafdb39b4e543938b3f3c8034b8d5
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1080a48bd5290ff2480aba802444f53e4fc0ca69
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d91b12925fd03a83546e0e89d0951816dc2e0721
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ko/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kok/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/kok/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e1d4892799b585e8da2230b0254f38ebf2f964f0
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kok/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kv/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/kv/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4244c580d0dbff6e0425684c1a5250920130074f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kv/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/kw/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/kw/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..97008fa814a0759c3ed724f0f5563d04abd94095
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/kw/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ky/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ky/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4167a65a7d420cb737a683a3116353794a5af3e1
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ky/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ky/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/ky/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..5d4ec7c15a86bb7937b54a033596def6451c35b7
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ky/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lo/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/lo/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4456314088fe5b8103e169ef1055a2199c430e66
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lo/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..b1faf83711e7e4abd4373e9bdd3f96bb1bd8bf6a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..694745b41d9145108657f04683c2f5492c952da7
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..500b1ac0c78dfcd4d068ac7ed6a5d73eb8e12718
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..dacbd390578f3ee1b643215d0c8badfea406af29
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1ac600b9da04c3a894393a96b88051b024d70c56
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..2a6a34567f81516db17579837f395dbbbb42ff3d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..295c093600d401eaca7525a2936238bad3b088bd
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lt/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..527603e4592b8b26a149dbb249066f255d536eb0
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3c45e02bdf1a11e6f81f917b1c28403aaebfc465
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..471a46ae41824bf1c18d24ad156dc3c6b0d3b13a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ac5dd3d99416ee61ebf27d87071b51cafb51b3cd
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e8d3fd7f9ee280e604f8c026e908987edc71aab5
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a0f56160f5640f80861bc8f5a27724b44d0456e8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/lv/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mai/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/mai/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fc6879afd9a00f4ad0ccbbf998b74916cf4270f5
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mai/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mhr/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/mhr/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f836ad394db98036e7ef5161c04052c387097f35
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mhr/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mi/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/mi/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..05a4454513f88976875d5904964d8a3de5e41b05
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mi/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mi/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/mi/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a31d9f3a02dc680eeb513764d567fc16abab24d7
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mi/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mi/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/mi/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..eb0a4af4d6a754411129f173f1a1996935f690a7
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mi/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mk/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/mk/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..9406e1cea3082814bedb4c1be611c173f56a2fa8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mk/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mk/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/mk/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..64a9addbaef6e6a43a35a952dc38c19f0fbcd036
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mk/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mk/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/mk/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1dc5ca3d9735d0c2fb736ff38c4b4c907ac09687
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mk/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ml/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/ml/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d169c8f101a5163dbe5455051431fea784e0c95e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ml/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ml/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ml/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e0f5730561ac46727029e392ec9aa1578a7d76c6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ml/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ml/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ml/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..96458b777f7fb16f2cf620f60fd74e85b24c3ef9
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ml/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mn/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/mn/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7f7e9e942af041ab9237a0c706719ec53f11bc09
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mn/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mn/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/mn/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..837ada1bb50d5049549df7ed2113d3f669631b8d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mn/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mn/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/mn/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..05173c1defef49a7d55a0df2bcefd8b74f677eff
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mn/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mn/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/mn/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..624cd7f71d6a8860418f7a3e40647e8a063103d7
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mn/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mr/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/mr/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..280515b9f578168f4252c7b98eaef7b36e917b47
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mr/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mr/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/mr/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..07692d71d1dbc5ebe456c1ff523c7e459f7cd102
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mr/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ms/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ms/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d283cf9e7829ac9d66e58f8194bc0b353cc16680
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ms/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ms/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ms/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..de59ce4aaf5fcef7fd260fd1152a72ef44d81b42
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ms/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ms/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/ms/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..5bf9e30eb3ad326023eacb63c80e0b4a2fbe54a1
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ms/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mt/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/mt/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..aff00c8604929b6af1130499cc9702edc436bffa
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mt/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mt/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/mt/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1f399179a07edc976be1a512c8af536f70c9b97d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mt/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/mt/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/mt/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..93027f3407257e73406faa7521450ddd46be9795
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/mt/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/my/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/my/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ae499dd61a62e96311a3cbee9da32dfe48036773
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/my/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/na/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/na/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..dc8ae9e8e3d5d9181313e956ccaac2ac0f1a2947
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/na/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nah/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/nah/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..acc076cebf768295f9b5e7731b32a0c2d77698a5
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nah/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..85b2c590e0b91e26a2c369059224510770b55825
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..6e66f986b98b6c225f26faa3835a1fe34c0d5c35
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..016ffa9d4fd52df8533e306a9cdb4aedbb533d42
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1346d45a9b88976202b980c420606aed138c4ff5
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e121d6c21f7dbd40afb815e295e8172322ba65fe
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..2b9049729be04fefbb9d7cfa173dc8a75c79326e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7c592281002a8e0413a34be70723a4c358c9e38d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nb_NO/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ne/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ne/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..48d0cdaa5a3720a2bc05fce8804429fab4051381
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ne/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ne/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ne/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..663609177c8a69ed1d6503bceb8bd68e5bf68fd2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ne/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1da79d79e27d6a6a7b7f3724007b9cc86e5abbe8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a2a8e8460023d71fd0676ad42bd8f995b89d858c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1fab74f4197b9550c71d75c136f8e18d83e328e2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..444308b5f0fade522dfd82934df332babc00fede
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3627a9b4084d69c5f2e0ba99144f4b754a6d65d3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7b15c671c230bb2c4c80e99b6fb28c6a457d2610
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nl/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d9721924c75810717827bddb3ded265c97e8cb2b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..03f1189292f75b1c45d61846ba4d427f78e07f26
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d848384273f3c991d2e71f7e6bd740d2984272dc
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d25f28bf966fb49dad8cfb27258ddd57a5be8ac0
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0c14311429542f71339a529266ceab66352786c3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nn/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nso/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/nso/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..01ae29908e7c5d66a48c422d53b24af806ae78e9
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nso/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nso/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/nso/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e82a04083768766f2ee853fefe5275aacb7b67be
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nso/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nso/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/nso/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ae8a2bc7df648c3c6e05f59b2f9421802bdda519
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nso/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nso/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/nso/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3e38f24beb54ab8c813adf0fc7a98fdada4d2fd2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nso/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/nv/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/nv/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..76fa0960aaff4f7e0b8ee2a354cdffe6b313f60d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/nv/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..153d4f23806ecf7a37d22377cef6ffc5fd26b95f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0f714223340a4eb36c20bd1ad70f2f4290ac5baf
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e038825f9756bdfe26313d4488dad3344c26dd9b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c710b36d76af8a54e38ed2d9d5f25fbcde076ebd
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..5031841ee10c134b82d7b06ec2124c6886a97aef
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..804edfa487cc9aad5cb976483c631371506de313
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ceae311081d0582194ce96ad8a3725269aa7ae16
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/oc/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/or/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/or/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..248de4c2c745f3a991e7b048692c07e7d3a7f422
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/or/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/or/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/or/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..dae51ec48bd75235d1d41a0dea1ca5ec7379de2c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/or/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pa/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/pa/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7efeea06a18c2e34893510cba4e7e5f45e3c1588
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pa/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pa/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/pa/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..6a5967fd401b11614fd5e4c86a0099ff1d9bc8b9
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pa/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pa_PK/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/pa_PK/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..395e4771b4663ada837b8a1ab6cabc1632e44d61
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pa_PK/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pa_PK/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/pa_PK/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8b32dbc9e7d35de6e15255d39200414296cf76a9
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pa_PK/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pa_PK/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/pa_PK/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..833a5ec82feb0517ce2e28420b91470c4bbae977
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pa_PK/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pap/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/pap/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..188a02cba646db91f67501f7508ecf3f1c7a3643
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pap/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pi/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/pi/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..725c7046c8401b43401f14aa95a8baaa9a566d80
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pi/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..cfb37d7af0c759c227f3ebeb920d8071d3c627ca
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e625675bb947392e3e7e922c362bcf719d3ce696
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..5bb1dc42d16369a7a1694ad3de22104b6caad429
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..065ce24eed9d74e3674eb0bf6f62d1e3aa0b591c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a71e86dabad9f3a58dc5c3f19da8cd9f3a3293e3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pl/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ps/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ps/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3c0d9eeca0e9f3ea69e545fee196333d813b7973
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ps/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ps/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ps/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f94fd29360ed5e68584c1e17f978b90880d0bb71
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ps/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ps/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/ps/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..99fb401b6756b848cac8a0194be9d447145190af
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ps/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..86b484020f4199259e03cce6fad9b664ba8b3c27
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c243a4aa1a636519070bc2bd0d25ffc54e13449a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..65911077113f86bb278263bdf5e6f8cd9bb6b7ea
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1e4bf45d1fbeab10adcd4ec1a5599ca2674b3119
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e4b946f1fbefc4f2a02b07184d96ab8cfb09773a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pt/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8a9e4b36b2cb6f6c266acd9fed306389fc366ca8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8fa300b6b1dc7ff78ad86dcee4002233cc54a5af
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..86598e253dc322382b35d031b0bdaadb410442b0
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..b2e3b7be8098637ddffb18b69d2cc641ac819abf
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..43991bb9f6561b4d3531de574be664cb7d963969
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..23af2d7d09a70efcde1819f57e399761f5f2fced
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..644fb5a0ad4384f7912feda219df2c5a5f275226
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/pt_BR/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8402eea4c4ae07462e1c4c3e500533471df1caa6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1392dc229193543519f11b03bdf40364906f7855
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..db1382d1b91b622d083db9fde2a505e3dec04cb8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d27a289c1f48233b70766c244049472f78cbb0ea
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a2fb70290ebb16de09c7e490c7618aed664ce306
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ba535226cd8128b3c64fe04a850980e4bb913004
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ro/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ro_MD/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ro_MD/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..35d1af72649c57bdd1bfdaa2b8cea458859ceae0
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ro_MD/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d254a0a4b1d46d452e8fee73ca845b2cfb3dca21
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0826877a2d505c2bd89b9cc6934d928c64dcdda0
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..9bf4d1aafd0b896ec12cc3d359cc4673706c730d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..aefa88b6e1bd183348a3c95b28b55de05a079ccf
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..b85ce6dcb6b3125acd8728136d47933d7576ea0b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4bcfd655cac03a3fe2a4c6e10f951c62edc137e7
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ru/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/rw/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/rw/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f805fad47f4f26f4852ddca20fd21c4d8cbf180f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/rw/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/rw/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/rw/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..318872b7271655a17538bf0709a603d163405bb6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/rw/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/rw/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/rw/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d74e51067550537843d3fbfb0c642ec6fc0352fc
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/rw/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/rw/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/rw/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..627a6c41ac9fd43bf686bd2f2a8b4d4b6349c5f2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/rw/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..72e254f1ca5c71a135efaf8d6d9fa9231b2d1ae7
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fb225ddf75cce31c844084ea886ba9251d5581d6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..6f7eeeb9d0bff780c926dee920a665b1da88c1f9
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d13663382e6cefc89e45b56e8c2d2b4c9b032c2c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..63c4dbff8f224d1af49ed036de6055477a580f02
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..019e17cc47be671310861e82aeb0fefbe268f827
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..9026d343b67164300819be3ceafa9d11fb182421
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sc/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sd/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/sd/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8ec898c8c02476850d4ef0082f2c96a0df858edf
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sd/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/si/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/si/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..6ddb6654b0fb5c9292fc8fd9c2032f3a115bf33d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/si/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/si/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/si/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..5b1afaf5a08b140df33446703bf84921fe09fc0c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/si/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/si/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/si/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7ab63cb4cdffc92170538cdf087db710bc24029e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/si/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..2c8fa141462ac39db0e1a00c05c0d1c22839a9f8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..16f61cde94b2f333b9a9360349d502c2f101616c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..6ffeb35af3814effa64af6a5e4fa939790129557
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..cc312a861e9360f4e1192a2779cd4b028568d637
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ffe0e99d3e14f48db5f726fb39c2145e03b3e733
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..11399ae65629a83115dfc0087f4bff7fc05245e2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sk/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..350c7539bc1099e4654deda5e2fa1f864bd571e0
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..132b59af8fcb3ace4989f21279aab20436c5b44f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..67e5bc37ea2572a3ee5c27c9c96a6465feb9b661
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..40acae6aa4497bfb7eb7440046ca68a1e59ca01a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4b6e8aeb75569e90fc12240e7f5178f6e2310e85
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..aef4a572f75c2a2150b53a4cf1a08086c3155e69
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sl/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7a7bfa63ac8b3eb243b04d6c2edb8f1f61cbc31d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..cef5d4d469dabc98b66f533b3e5cf05fbf835c9e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..52262df100c1764d66440d5124d2afc7fbf6c626
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..708e7c351d1c92b2c19f40d8a5dfff91974c4d9a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..dc44d0cccbd4348daea0f50411b575cd4d896ac9
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..cfe9bb0b8489796b50c1107b5020682ef34dd75c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/so/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/son/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/son/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..77335892fbad9ce16165ccf723b41f4766555e21
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/son/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..436250eb6f10df3041c8af3a8bf0a4e98c96f32f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4ac70ed01ce8f6008311461afcdac4fc01d3b055
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8c41c3e94d0fc2661f7e3f571b8ebf6a0cdff849
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d715df006349a36d2926ac89f6a6068640089037
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..baade15f8502da2591d51a06c7bcfc116aaee88e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a0b9ef86ecd0c335cfc71b51b4672e2600564485
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..be035d481ab56775cf8a30697ba592c6ec7126f9
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sq/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..533c89ea74d43f0be8ee25fd4bea2051ce94348e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a1e016ae84932a60575afefd2bbcd94b8ca0fc1d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7487e6fd60138e7bbabb1d076b46e4002807c1e2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..acc25f037057df27dda5877f707a481d69f9b989
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a20ae0c9a880c028f1a73a77f877ce47cce55931
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a0d81f85484a93ce4982bcbf6e67b7e6ba7012f5
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sr/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7b613906ab16e4caea774f92610196d44ee5187a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..77b4e28181091c35e18b47e868e1bda19c40f3d9
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f4d432635023835c2e3836055b13557eccb87a35
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f8ad12343c3a60b5fcd831b1539286ad5ee413e4
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ddf4b59908b31f116b0122c0e49ba6ee23bf6b71
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d486942836f412bce4e6484b1f61d96a195f69c0
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sr@latin/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e1f8261a5c6005e26cd7fa4674f5e7c080d3d015
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..362b3be27d0e7f7cfc905c0e358628c4f207c15a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..974ba8c7591ca466651cd8471a2851f1ece85504
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..cb6f447f2f3cb71b6930df77ec2ec2e6ee53f60f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d4a0407b51727c142b5823d7d77b8a0bcb410341
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sv/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sw/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/sw/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..62bb04e086f04575dee9f649da41143d8f978b33
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sw/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/sw/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/sw/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fda2f665bf4c23c93c15e1e07dec481a4f310507
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/sw/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d71800501176f0d8cc5f83b58b8fb5d1178f0c0d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0e98c7f3bddb4e49c480574d467eeff91a1b56de
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c27fdce88fb69a446bf23fde52ea88cf18918bba
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..257950a8c60d9b007781e5bd694a4125208377bb
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a3a66cdb5d8fc4b21992d2477ff89b44b9f75779
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..39ae6fcb9accd7060c5fd2e22faa3c3f0a49ec26
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ta/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/te/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/te/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8aa97b6bf743003bd34b48cd06629c1d6732db98
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/te/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/te/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/te/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..7bd70f54ed4069ce5e07ac365397e959eebf3362
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/te/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tg/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/tg/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a410182bd9cfcae66fc8ccc729dc7fbac08c2da6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tg/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8e32df9af4120094563d8fdaf0155a18dc5a99da
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..68df29557a1f0e7e2b968d66a07ee26295a15d50
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..b556ec94fb64d6a797f3bee6b38eef995a00a005
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..72f87038e054cbda49f8b5a9151ac2fb713b1a81
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c2f3b0b4efe7dd25b2ccc8624252a4405c13631b
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..2af06ce1774840cfb01a0a427bc96910dff89451
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/th/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ti/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ti/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..2074d1ad2885bf762222f30358d7397c7d5a479e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ti/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ti/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ti/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c3ce62f1bfff153ebba6e04d4993ba359b189c03
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ti/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ti/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/ti/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a8830d4897f41a279eced2e38859a8ae990f73e4
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ti/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tig/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/tig/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ae2791a14f54c2712484222425af843cf7e73c60
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tig/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tig/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/tig/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fb92fd70daf06525562d1d47a5f83749a0a491e6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tig/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tig/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/tig/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..493a893a9fbe61fc70d8cc8f7524064ca6006f88
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tig/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tk/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/tk/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..6ace06e01d856d65e1015a339cf16e4a86fcd761
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tk/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tk/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/tk/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fd0f3631a157885cf2627e11dad3905dc844dc5a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tk/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tl/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/tl/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..005f440d734bf3e22b2b23919916a080d8121523
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tl/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tl/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/tl/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..89bef01cb142269a425f51538670656ad9f026b6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tl/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a97e6cf6e23033005592921154998300e7deb751
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fd9db79e2e3f05d17745d0322947d16bae7e9f86
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4cf52fb943889b40c0ae8a25f9878f9ceb82c7c6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..748fc81e3d471cb49f0889feb6e889b7f75497d5
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e29a5bc802698eeec59caae92b9cd0acbabde590
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..bf0639b75020c8812f4b116ac50013a27a24b56d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tr/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tt/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/tt/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..78cd5739bb9663276c4b1adc180eb318090c1ad9
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tt/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tt/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/tt/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ca0ea7d5f096f5c074ee0bbec74202fb71b0d823
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tt/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tt/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/tt/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..20098623f32ffb73abd4ede8e38bdf85b2bffa6f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tt/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tt@iqtelif/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/tt@iqtelif/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..683c9e1cc0279df5b397ef962eb6ba48138b6a12
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tt@iqtelif/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tt@iqtelif/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/tt@iqtelif/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..91231b93464e7e4d7006fd5e36025320518bfbcf
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tt@iqtelif/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tt@iqtelif/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/tt@iqtelif/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4e5fe1fe81902e362e8ce9550d55e2266c851948
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tt@iqtelif/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tzm/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/tzm/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e7bf3ddbfc8b4991d6d0407e646c737e9d75a8d3
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tzm/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tzm/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/tzm/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..854b17abf368de2935328b95a8b2a102ea5a060a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tzm/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/tzm/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/tzm/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a690e2b372ca9c486c7bfd5e3f1b4cba2e9afbff
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/tzm/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ug/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ug/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3f840cb1e8c803af0d26839911dd69eba0ebf6fb
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ug/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ug/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ug/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..289ac315c19582e4a161b89d72fb61249ef8ba9f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ug/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d6d140da062f0426bd1708fab2f2e060f8f6e2c4
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8055991f0e5913744e24e63aaea18b4d7ce08bc5
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..fed97dad0043a9ec631861073f1259c665ccef6c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..830147c23ff0c37f3786d7130521a642090cfae8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..561addea87419b2df5f9be11d4cf0b509547a225
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/uk/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ur/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ur/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d66e58af509adc1754f32416de3b3e22fd2e8d7a
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ur/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/uz/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/uz/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d4a1ed1a18e20a661fc4a3fb0e7f02655cc97a99
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/uz/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ve/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/ve/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..57c2cb5683d197b8fdbec7e36808c5f29758c66e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ve/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ve/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/ve/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..3188326058e030fa87789464ed1d60c770f937fc
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ve/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ve/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/ve/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..ce6b77f7e513419724f9702a73f333c329ea6b50
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ve/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/ve/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/ve/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..64e2747b1a276552a7c664d7e702b34af0aac6be
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/ve/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..948c4653864e2d95ae58c01a89ac0f203aff815d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..72be6efa2617b25d539e0232cf895cf914c74198
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..cdd22fd797027acfcf7f2447a1ab9fdf525cc62c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a3a95b99a62314a8d8f7c7198470034ae2b2721d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..5efdd0af7857f56290f618c8168da93f67b84b0c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/vi/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/wa/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/wa/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f146763d908bbe7c28395331eef9565634c981e2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/wa/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/wa/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/wa/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..5e0f893b0fb829e091b504633efb123ffe18cf28
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/wa/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/wa/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/wa/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..27792c762c64256980e60d05125dac70d434c146
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/wa/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/wa/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/wa/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..098797def0ca9614f5be1c617aa8fafdb12d5de4
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/wa/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/wal/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/wal/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8e0f5ccf89349f073ca8eaf1156cb70e52661edf
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/wal/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/wal/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/wal/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..faa48c98da2eaeeae8eba38b0ade55feff1952e5
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/wal/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/wo/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/wo/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4fffe93a0242918d2e6f0e48cbf4114530a5e502
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/wo/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/wo/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/wo/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..cd165471adb93f9bc281b2acafece8547547bea1
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/wo/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/xh/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/xh/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..f5ead56813a06d22790fad02e6188a3025428567
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/xh/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/xh/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/xh/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..b8d832ed4603583d58f3b615486ddf0cecd2d618
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/xh/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/xh/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/xh/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c41ef44fde55fc56fd5d4d384fd222eb9daa9459
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/xh/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/yo/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/yo/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..8038e0c9831463e5efe08a395faa971c71eed204
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/yo/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c18be2a4a123a633fd1c8e6d2b43935a6bff7538
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0e3ae2e7ab3104f2fd5f1c5c94fc2f210e765de2
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..d8259f928464b051bc49cc51b36659984b57eaaf
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..74c3b04ee69531ae4fbfa8b5c2904fc904eb1da0
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..af8ec26f6e7c47d7501a8297d402dafc1ec9a5ff
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_CN/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_HK/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/zh_HK/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a7bbc5e8f6e9adb03f347a3b6e6c6b797dbb3320
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_HK/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_HK/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/zh_HK/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..4c70d68d2d88f68a19c1f2445ad391c49ccd089e
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_HK/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_HK/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/zh_HK/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..5fbab85d372362a444ddfd61b605abed01e78bf8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_HK/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_HK/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/zh_HK/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..33372520ce3574bf711bf173eed979463c067473
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_HK/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_Hans/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/zh_Hans/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..770b809f6f6c4483bb84df9ac39fbe573ae9cfaa
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_Hans/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_Hant/LC_MESSAGES/iso639-5.mo b/lib/python3.1/site-packages/pycountry/locales/zh_Hant/LC_MESSAGES/iso639-5.mo
new file mode 100644
index 0000000000000000000000000000000000000000..a9ed595c2e37e441a4b33e394ea6de425a2989ae
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_Hant/LC_MESSAGES/iso639-5.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso15924.mo b/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso15924.mo
new file mode 100644
index 0000000000000000000000000000000000000000..1e577c42252b27232107f3292df985eef6a70725
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso15924.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..c2ff70edfb5503ba9f836dc6cabfa5a448faed90
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso3166-2.mo b/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso3166-2.mo
new file mode 100644
index 0000000000000000000000000000000000000000..331d34cbbe4e5b7b176877e9e0eb87dc6d0a4b78
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso3166-2.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0aa9d65a2047541d905954f5b1c41e62d1f84ce8
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso4217.mo b/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso4217.mo
new file mode 100644
index 0000000000000000000000000000000000000000..67454c226e4f4a52e4f4b2668f05444e15580d9c
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso4217.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..cb224bce00d5b385ff18fee316ada672559c261f
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zh_TW/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zu/LC_MESSAGES/iso3166-1.mo b/lib/python3.1/site-packages/pycountry/locales/zu/LC_MESSAGES/iso3166-1.mo
new file mode 100644
index 0000000000000000000000000000000000000000..e85766cd18f82e2fbf9b38bf1e9c31698128634d
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zu/LC_MESSAGES/iso3166-1.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zu/LC_MESSAGES/iso3166-3.mo b/lib/python3.1/site-packages/pycountry/locales/zu/LC_MESSAGES/iso3166-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0812046537cb85d6ab0bb288fedb50e7e5b6ffde
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zu/LC_MESSAGES/iso3166-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/locales/zu/LC_MESSAGES/iso639-3.mo b/lib/python3.1/site-packages/pycountry/locales/zu/LC_MESSAGES/iso639-3.mo
new file mode 100644
index 0000000000000000000000000000000000000000..0165ee36813b84015d6a73ce1cd9c5186abc54df
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/locales/zu/LC_MESSAGES/iso639-3.mo differ
diff --git a/lib/python3.1/site-packages/pycountry/py.typed b/lib/python3.1/site-packages/pycountry/py.typed
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lib/python3.1/site-packages/pycountry/tests/__pycache__/test_general.cpython-310.pyc b/lib/python3.1/site-packages/pycountry/tests/__pycache__/test_general.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1830abe8eab7122b4309b862dafe3b41f7f0fdf6
Binary files /dev/null and b/lib/python3.1/site-packages/pycountry/tests/__pycache__/test_general.cpython-310.pyc differ
diff --git a/lib/python3.1/site-packages/pycountry/tests/test_general.py b/lib/python3.1/site-packages/pycountry/tests/test_general.py
new file mode 100644
index 0000000000000000000000000000000000000000..7039f1d89a4ad06bb6ebbd7759723f01aee43de5
--- /dev/null
+++ b/lib/python3.1/site-packages/pycountry/tests/test_general.py
@@ -0,0 +1,468 @@
+import gettext
+import os.path
+import re
+from importlib import metadata as _importlib_metadata
+from unittest.mock import patch
+
+import pytest
+
+import pycountry
+import pycountry.db
+
+
+@pytest.fixture
+def countries():
+    pycountry.countries._clear()
+    yield pycountry.countries
+    pycountry.countries._clear()
+
+
+def test_country_list(countries):
+    assert len(pycountry.countries) == 249
+    assert isinstance(list(pycountry.countries)[0], pycountry.db.Data)
+
+
+def test_country_fuzzy_search(countries):
+    results = pycountry.countries.search_fuzzy("England")
+    assert len(results) == 1
+    assert results[0] == pycountry.countries.get(alpha_2="GB")
+
+    # Match alternative names exactly and thus NL ends up with
+    # "Sint Maarten" before SX with "Sint Maarten (Dutch part)"
+    results = pycountry.countries.search_fuzzy("Sint Maarten")
+    assert len(results) == 2
+    assert results[0] == pycountry.countries.get(alpha_2="NL")
+    assert results[1] == pycountry.countries.get(alpha_2="SX")
+
+    # Match with accents removed, first a country with a partial match in the
+    # country name, then a country with multiple subdivision partial matches,
+    # and then a country with a single subdivision match.
+    results = pycountry.countries.search_fuzzy("Cote")
+    assert len(results) == 3
+    assert results[0] == pycountry.countries.get(alpha_2="CI")
+    assert results[1] == pycountry.countries.get(alpha_2="FR")
+    assert results[2] == pycountry.countries.get(alpha_2="HN")
+
+    # A somewhat carefully balanced point system allows for a (bias-based)
+    # graceful sorting of common substrings being used in multiple matches:
+    results = pycountry.countries.search_fuzzy("New")
+    assert results[0] == pycountry.countries.get(alpha_2="NC")
+    assert results[1] == pycountry.countries.get(alpha_2="NZ")
+    assert results[2] == pycountry.countries.get(alpha_2="PG")
+    assert results[3] == pycountry.countries.get(alpha_2="GB")
+    assert results[4] == pycountry.countries.get(alpha_2="US")
+    assert results[5] == pycountry.countries.get(alpha_2="CA")
+    assert results[6] == pycountry.countries.get(alpha_2="AU")
+    assert results[7] == pycountry.countries.get(alpha_2="BS")
+    assert results[8] == pycountry.countries.get(alpha_2="TW")
+    assert results[9] == pycountry.countries.get(alpha_2="MH")
+
+    # bug #34, likely about capitalization that was broken
+    results = pycountry.countries.search_fuzzy("united states of america")
+    assert len(results) == 1
+    assert results[0] == pycountry.countries.get(alpha_2="US")
+
+
+def test_historic_country_fuzzy_search(countries):
+    results = pycountry.historic_countries.search_fuzzy("burma")
+    assert len(results) == 1
+    assert results[0] == pycountry.historic_countries.get(alpha_4="BUMM")
+
+
+def test_germany_has_all_attributes(countries):
+    germany = pycountry.countries.get(alpha_2="DE")
+    assert germany.alpha_2 == "DE"
+    assert germany.alpha_3 == "DEU"
+    assert germany.numeric == "276"
+    assert germany.name == "Germany"
+    assert germany.official_name == "Federal Republic of Germany"
+
+
+def test_missing_common_official(countries):
+    aruba = pycountry.countries.get(alpha_2="AW")
+    assert aruba.alpha_2 == "AW"
+    assert aruba.name == "Aruba"
+    with pytest.raises(AttributeError, match="official_name"):
+        aruba.official_name
+    with pytest.raises(AttributeError, match="common_name"):
+        aruba.common_name
+
+
+def test_missing_common_official_use_different(countries):
+    vietnam = pycountry.countries.get(alpha_2="VN")
+    assert vietnam.alpha_2 == "VN"
+    assert vietnam.name == "Viet Nam"
+    assert vietnam.official_name == "Socialist Republic of Viet Nam"
+    assert vietnam.common_name == "Vietnam"
+
+
+def test_country_missing_attribute(countries):
+    germany = pycountry.countries.get(alpha_2="DE")
+    with pytest.raises(AttributeError):
+        _ = germany.foo
+
+
+def test_subdivisions_directly_accessible(countries):
+    assert len(pycountry.subdivisions) == 5046
+    assert isinstance(list(pycountry.subdivisions)[0], pycountry.db.Data)
+
+    de_st = pycountry.subdivisions.get(code="DE-ST")
+    assert de_st.code == "DE-ST"
+    assert de_st.name == "Sachsen-Anhalt"
+    assert de_st.type == "Land"
+    assert de_st.parent is None
+    assert de_st.parent_code is None
+    assert de_st.country is pycountry.countries.get(alpha_2="DE")
+
+
+def test_subdivisions_have_subdivision_as_parent():
+    fr_01 = pycountry.subdivisions.get(code="FR-01")
+    assert fr_01.code == "FR-01"
+    assert fr_01.name == "Ain"
+    assert fr_01.type == "Metropolitan department"
+    assert fr_01.parent_code == "FR-ARA"
+    assert fr_01.parent is pycountry.subdivisions.get(code="FR-ARA")
+    assert fr_01.parent.name == "Auvergne-Rhône-Alpes"
+
+
+def test_query_subdivisions_of_country():
+    assert len(pycountry.subdivisions.get(country_code="DE")) == 16
+    assert len(pycountry.subdivisions.get(country_code="US")) == 57
+
+
+def test_scripts():
+    assert len(pycountry.scripts) == 182
+    assert isinstance(list(pycountry.scripts)[0], pycountry.db.Data)
+
+    latin = pycountry.scripts.get(name="Latin")
+    assert latin.alpha_4 == "Latn"
+    assert latin.name == "Latin"
+    assert latin.numeric == "215"
+
+
+def test_currencies():
+    assert len(pycountry.currencies) == 181
+    assert isinstance(list(pycountry.currencies)[0], pycountry.db.Data)
+
+    argentine_peso = pycountry.currencies.get(alpha_3="ARS")
+    assert argentine_peso.alpha_3 == "ARS"
+    assert argentine_peso.name == "Argentine Peso"
+    assert argentine_peso.numeric == "032"
+
+
+def test_languages():
+    assert len(pycountry.languages) == 7910
+    assert isinstance(list(pycountry.languages)[0], pycountry.db.Data)
+
+    aragonese = pycountry.languages.get(alpha_2="an")
+    assert aragonese.alpha_2 == "an"
+    assert aragonese.alpha_3 == "arg"
+    assert aragonese.name == "Aragonese"
+
+    bengali = pycountry.languages.get(alpha_2="bn")
+    assert bengali.name == "Bengali"
+    assert bengali.common_name == "Bangla"
+
+    # this tests the slow search path in lookup()
+    bengali2 = pycountry.languages.lookup("bAngLa")
+    assert bengali2 == bengali
+
+
+def test_language_families():
+    assert len(pycountry.language_families) == 115
+    assert isinstance(list(pycountry.language_families)[0], pycountry.db.Data)
+
+    aragonese = pycountry.languages.get(alpha_3="arg")
+    assert aragonese.alpha_3 == "arg"
+    assert aragonese.name == "Aragonese"
+
+
+def test_locales():
+    german = gettext.translation(
+        "iso3166-1", pycountry.LOCALES_DIR, languages=["de"]
+    )
+    assert german.gettext("Germany") == "Deutschland"
+
+
+def test_removed_countries():
+    ussr = pycountry.historic_countries.get(alpha_3="SUN")
+    assert isinstance(ussr, pycountry.db.Data)
+    assert ussr.alpha_4 == "SUHH"
+    assert ussr.alpha_3 == "SUN"
+    assert ussr.name == "USSR, Union of Soviet Socialist Republics"
+    assert ussr.withdrawal_date == "1992-08-30"
+
+
+def test_repr(countries):
+    assert re.match(
+        "Country\\(alpha_2=u?'DE', "
+        "alpha_3=u?'DEU', "
+        "flag='..', "
+        "name=u?'Germany', "
+        "numeric=u?'276', "
+        "official_name=u?'Federal Republic of Germany'\\)",
+        repr(pycountry.countries.get(alpha_2="DE")),
+    )
+
+
+def test_dict(countries):
+    country = pycountry.countries.get(alpha_2="DE")
+    exp = {
+        "alpha_2": "DE",
+        "alpha_3": "DEU",
+        "name": "Germany",
+        "numeric": "276",
+        "official_name": "Federal Republic of Germany",
+        "flag": country.flag,
+    }
+    assert dict(country) == exp
+
+
+def test_dir(countries):
+    germany = pycountry.countries.get(alpha_2="DE")
+    for n in "alpha_2", "alpha_3", "name", "numeric", "official_name":
+        assert n in dir(germany)
+
+
+def test_get(countries):
+    c = pycountry.countries
+    with pytest.raises(TypeError):
+        c.get(alpha_2="DE", alpha_3="DEU")
+    assert c.get(alpha_2="DE") == c.get(alpha_3="DEU")
+    assert c.get(alpha_2="Foo") is None
+    tracer = object()
+    assert c.get(alpha_2="Foo", default=tracer) is tracer
+
+
+def test_lookup(countries):
+    c = pycountry.countries
+    g = c.get(alpha_2="DE")
+    assert g == c.get(alpha_2="de")
+    assert g == c.lookup("de")
+    assert g == c.lookup("DEU")
+    assert g == c.lookup("276")
+    assert g == c.lookup("germany")
+    assert g == c.lookup("Federal Republic of Germany")
+    # try a generated field
+    bqaq = pycountry.historic_countries.get(alpha_4="BQAQ")
+    assert bqaq == pycountry.historic_countries.lookup("atb")
+    german = pycountry.languages.get(alpha_2="de")
+    assert german == pycountry.languages.lookup("De")
+    euro = pycountry.currencies.get(alpha_3="EUR")
+    assert euro == pycountry.currencies.lookup("euro")
+    latin = pycountry.scripts.get(name="Latin")
+    assert latin == pycountry.scripts.lookup("latn")
+    fr_ara = pycountry.subdivisions.get(code="FR-ARA")
+    assert fr_ara == pycountry.subdivisions.lookup("fr-ara")
+    with pytest.raises(LookupError):
+        pycountry.countries.lookup("bogus country")
+    with pytest.raises(LookupError):
+        pycountry.countries.lookup(12345)
+    with pytest.raises(LookupError):
+        pycountry.countries.get(alpha_2=12345)
+
+
+def test_subdivision_parent():
+    s = pycountry.subdivisions
+    sd = s.get(code="CV-BV")
+    assert sd.parent_code == "CV-B"
+    assert sd.parent is s.get(code=sd.parent_code)
+
+
+def test_subdivision_missing_code_raises_keyerror():
+    s = pycountry.subdivisions
+    assert s.get(code="US-ZZ") is None
+
+
+def test_subdivision_empty_list():
+    s = pycountry.subdivisions
+    assert len(s.get(country_code="DE")) == 16
+    assert len(s.get(country_code="JE")) == 0
+    assert s.get(country_code="FOOBAR") is None
+
+
+def test_has_version_attribute():
+    try:
+        _importlib_metadata.distribution("pycountry")
+    except _importlib_metadata.PackageNotFoundError:
+        pytest.skip("pycountry not installed correctly, you're on your own")
+    assert pycountry.__version__ != "n/a"
+    assert len(pycountry.__version__) >= 5
+    assert "." in pycountry.__version__
+
+
+def test_is_instance_of_language():
+    assert isinstance(pycountry.languages, pycountry.Languages)
+
+
+def test_is_instance_of_country(countries):
+    united_states = pycountry.countries.get(alpha_2="US")
+    class_name = united_states.__class__.__name__
+    assert class_name == "Country"
+
+
+def test_is_instance_of_subdivision():
+    assert isinstance(pycountry.subdivisions, pycountry.Subdivisions)
+
+
+def test_is_instance_of_script():
+    assert isinstance(pycountry.scripts, pycountry.Scripts)
+
+
+def test_is_instance_of_currency():
+    assert isinstance(pycountry.currencies, pycountry.Currencies)
+
+
+def test_add_entry(countries):
+    assert pycountry.countries.get(alpha_2="XK") is None
+
+    pycountry.countries.add_entry(
+        alpha_2="XK", alpha_3="XXK", name="Kosovo", numeric="926"
+    )
+
+    country = pycountry.countries.get(alpha_2="XK")
+    assert isinstance(country, pycountry.countries.data_class)
+
+
+def test_remove_entry(countries):
+    assert pycountry.countries.get(alpha_2="DE") is not None
+
+    pycountry.countries.remove_entry(alpha_2="DE")
+
+    assert pycountry.countries.get(alpha_2="DE") is None
+
+
+def test_remove_non_existent_entry():
+    with pytest.raises(KeyError, match="not found"):
+        pycountry.countries.remove_entry(name="Not A Real Country")
+
+
+def test_no_results_lookup_error(countries):
+    try:
+        import importlib_resources  # type: ignore
+    except ModuleNotFoundError:
+        from importlib import resources as importlib_resources
+
+    def resource_filename(package_or_requirement, resource_name):
+        return str(
+            importlib_resources.files(package_or_requirement) / resource_name
+        )
+
+    DATABASE_DIR = resource_filename("pycountry", "databases")
+    countries = pycountry.ExistingCountries(
+        os.path.join(DATABASE_DIR, "iso3166-1.json")
+    )
+
+    query = "nonexistent query"
+    with pytest.raises(LookupError):
+        countries.search_fuzzy(query)
+
+
+def test_subdivision_fuzzy_search_match():
+    results = pycountry.subdivisions.search_fuzzy("Alabama")
+    assert len(results) == 1
+    assert results[0].name == "Alabama"
+
+
+def test_subdivision_fuzzy_search_partial_match():
+    results = pycountry.subdivisions.search_fuzzy("Massachusett")
+    assert len(results) == 1
+    assert results[0].name == "Massachusetts"
+
+
+def test_subdivision_match():
+    results = pycountry.subdivisions.match("Alabama")
+    assert len(results) == 1
+    assert results[0].name == "Alabama"
+
+
+def test_subdivision_partial_match():
+    results = pycountry.subdivisions.partial_match("Massachusett")
+    assert len(results) == 1
+    assert results[0].name == "Massachusetts"
+
+
+def test_non_country_attribute_error():
+    english = pycountry.languages.get(name="English")
+    with pytest.raises(AttributeError):
+        english.official_name
+
+
+def test_country_attribute_error(countries):
+    canada = pycountry.countries.get(alpha_2="CA")
+    with pytest.raises(AttributeError):
+        canada.maple_syrup
+
+
+def test_with_accents():
+    assert pycountry.remove_accents("Café") == "Cafe"
+    assert pycountry.remove_accents("résumé") == "resume"
+    assert pycountry.remove_accents("naïve") == "naive"
+    assert pycountry.remove_accents("São Paulo") == "Sao Paulo"
+
+
+def test_without_accents():
+    assert pycountry.remove_accents("apple") == "apple"
+    assert pycountry.remove_accents("banana") == "banana"
+
+
+def test_empty_string():
+    assert pycountry.remove_accents("") == ""
+
+
+def test_special_characters():
+    assert pycountry.remove_accents("!@#$%^&*()") == "!@#$%^&*()"
+
+
+def test_unicode_characters():
+    assert pycountry.remove_accents("你好") == "你好"  # Chinese characters
+    assert (
+        pycountry.remove_accents("こんにちは") == "こんにちは"
+    )  # Japanese characters
+
+
+def test_subdivision_search_fuzzy_non_existent_subdivision():
+    with pytest.raises(LookupError):
+        pycountry.subdivisions.search_fuzzy("Non Existent Subdivision")
+
+
+def test_subdivision_partial_match_non():
+    result = pycountry.subdivisions.partial_match("Non Existent Subdivision")
+    assert len(result) == 0
+
+
+def test_subdivision_match_non():
+    result = pycountry.subdivisions.match("Non Existent Subdivision")
+    assert len(result) == 0
+
+
+def test_get_version_with_package_not_found():
+    # Mock importlib.metadata.version to raise PackageNotFoundError
+    with patch(
+        "importlib.metadata.version",
+        side_effect=_importlib_metadata.PackageNotFoundError,
+    ):
+        # Call get_version with a package name that doesn't exist
+        result = pycountry.get_version("non_existent_package")
+
+        # Assert that the result is 'n/a'
+        assert result == "n/a"
+
+
+def test_all_subdivisions_have_name_attribute():
+    subdivisions = pycountry.subdivisions
+    has_name_attr = [
+        hasattr(subdivision, "name") for subdivision in subdivisions
+    ]
+    all_have_name_attr = all(has_name_attr)
+
+    assert all_have_name_attr
+
+
+def test_subdivisions_with_missing_parents():
+    result = [
+        (i.code, i.parent_code)
+        for i in pycountry.subdivisions
+        if i.parent_code and not i.parent
+    ]
+    assert result == []
diff --git a/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/INSTALLER b/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/INSTALLER
new file mode 100644
index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68
--- /dev/null
+++ b/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/METADATA b/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..9b98dfcbf13183cfd2fe29a26d56981240e1f63d
--- /dev/null
+++ b/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/METADATA
@@ -0,0 +1,160 @@
+Metadata-Version: 2.4
+Name: pydantic_core
+Version: 2.33.2
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Rust
+Classifier: Framework :: Pydantic
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: MacOS
+Classifier: Typing :: Typed
+Requires-Dist: typing-extensions>=4.6.0,!=4.7.0
+License-File: LICENSE
+Summary: Core functionality for Pydantic validation and serialization
+Home-Page: https://github.com/pydantic/pydantic-core
+Author-email: Samuel Colvin , Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com>, David Montague , David Hewitt , Sydney Runkle , Victorien Plot 
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/pydantic/pydantic-core
+Project-URL: Funding, https://github.com/sponsors/samuelcolvin
+Project-URL: Source, https://github.com/pydantic/pydantic-core
+
+# pydantic-core
+
+[![CI](https://github.com/pydantic/pydantic-core/workflows/ci/badge.svg?event=push)](https://github.com/pydantic/pydantic-core/actions?query=event%3Apush+branch%3Amain+workflow%3Aci)
+[![Coverage](https://codecov.io/gh/pydantic/pydantic-core/branch/main/graph/badge.svg)](https://codecov.io/gh/pydantic/pydantic-core)
+[![pypi](https://img.shields.io/pypi/v/pydantic-core.svg)](https://pypi.python.org/pypi/pydantic-core)
+[![versions](https://img.shields.io/pypi/pyversions/pydantic-core.svg)](https://github.com/pydantic/pydantic-core)
+[![license](https://img.shields.io/github/license/pydantic/pydantic-core.svg)](https://github.com/pydantic/pydantic-core/blob/main/LICENSE)
+
+This package provides the core functionality for [pydantic](https://docs.pydantic.dev) validation and serialization.
+
+Pydantic-core is currently around 17x faster than pydantic V1.
+See [`tests/benchmarks/`](./tests/benchmarks/) for details.
+
+## Example of direct usage
+
+_NOTE: You should not need to use pydantic-core directly; instead, use pydantic, which in turn uses pydantic-core._
+
+```py
+from pydantic_core import SchemaValidator, ValidationError
+
+
+v = SchemaValidator(
+    {
+        'type': 'typed-dict',
+        'fields': {
+            'name': {
+                'type': 'typed-dict-field',
+                'schema': {
+                    'type': 'str',
+                },
+            },
+            'age': {
+                'type': 'typed-dict-field',
+                'schema': {
+                    'type': 'int',
+                    'ge': 18,
+                },
+            },
+            'is_developer': {
+                'type': 'typed-dict-field',
+                'schema': {
+                    'type': 'default',
+                    'schema': {'type': 'bool'},
+                    'default': True,
+                },
+            },
+        },
+    }
+)
+
+r1 = v.validate_python({'name': 'Samuel', 'age': 35})
+assert r1 == {'name': 'Samuel', 'age': 35, 'is_developer': True}
+
+# pydantic-core can also validate JSON directly
+r2 = v.validate_json('{"name": "Samuel", "age": 35}')
+assert r1 == r2
+
+try:
+    v.validate_python({'name': 'Samuel', 'age': 11})
+except ValidationError as e:
+    print(e)
+    """
+    1 validation error for model
+    age
+      Input should be greater than or equal to 18
+      [type=greater_than_equal, context={ge: 18}, input_value=11, input_type=int]
+    """
+```
+
+## Getting Started
+
+You'll need rust stable [installed](https://rustup.rs/), or rust nightly if you want to generate accurate coverage.
+
+With rust and python 3.9+ installed, compiling pydantic-core should be possible with roughly the following:
+
+```bash
+# clone this repo or your fork
+git clone git@github.com:pydantic/pydantic-core.git
+cd pydantic-core
+# create a new virtual env
+python3 -m venv env
+source env/bin/activate
+# install dependencies and install pydantic-core
+make install
+```
+
+That should be it, the example shown above should now run.
+
+You might find it useful to look at [`python/pydantic_core/_pydantic_core.pyi`](./python/pydantic_core/_pydantic_core.pyi) and
+[`python/pydantic_core/core_schema.py`](./python/pydantic_core/core_schema.py) for more information on the python API,
+beyond that, [`tests/`](./tests) provide a large number of examples of usage.
+
+If you want to contribute to pydantic-core, you'll want to use some other make commands:
+* `make build-dev` to build the package during development
+* `make build-prod` to perform an optimised build for benchmarking
+* `make test` to run the tests
+* `make testcov` to run the tests and generate a coverage report
+* `make lint` to run the linter
+* `make format` to format python and rust code
+* `make` to run `format build-dev lint test`
+
+## Profiling
+
+It's possible to profile the code using the [`flamegraph` utility from `flamegraph-rs`](https://github.com/flamegraph-rs/flamegraph). (Tested on Linux.) You can install this with `cargo install flamegraph`.
+
+Run `make build-profiling` to install a release build with debugging symbols included (needed for profiling).
+
+Once that is built, you can profile pytest benchmarks with (e.g.):
+
+```bash
+flamegraph -- pytest tests/benchmarks/test_micro_benchmarks.py -k test_list_of_ints_core_py --benchmark-enable
+```
+The `flamegraph` command will produce an interactive SVG at `flamegraph.svg`.
+
+## Releasing
+
+1. Bump package version locally. Do not just edit `Cargo.toml` on Github, you need both `Cargo.toml` and `Cargo.lock` to be updated.
+2. Make a PR for the version bump and merge it.
+3. Go to https://github.com/pydantic/pydantic-core/releases and click "Draft a new release"
+4. In the "Choose a tag" dropdown enter the new tag `v` and select "Create new tag on publish" when the option appears.
+5. Enter the release title in the form "v "
+6. Click Generate release notes button
+7. Click Publish release
+8. Go to https://github.com/pydantic/pydantic-core/actions and ensure that all build for release are done successfully.
+9. Go to https://pypi.org/project/pydantic-core/ and ensure that the latest release is published.
+10. Done 🎉
+
diff --git a/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/RECORD b/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/RECORD
new file mode 100644
index 0000000000000000000000000000000000000000..27d64c643e275f6c08a81c2ee89cdd026ff440f3
--- /dev/null
+++ b/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/RECORD
@@ -0,0 +1,12 @@
+pydantic_core-2.33.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+pydantic_core-2.33.2.dist-info/METADATA,sha256=78lBoOZz4Kzfzz_yMI_qFHMFs2SE3VgnWpeGPtjycKs,6757
+pydantic_core-2.33.2.dist-info/RECORD,,
+pydantic_core-2.33.2.dist-info/WHEEL,sha256=vZCrFUqux3RT3x-5MYcfGpyb4lW1eii-LV1Nx1Zs4nE,129
+pydantic_core-2.33.2.dist-info/licenses/LICENSE,sha256=Kv3TDVS01itvSIprzBVG6E7FBh8T9CCcA9ASNIeDeVo,1080
+pydantic_core/__init__.py,sha256=TzOWuJMgpXaZcPiS2Yjd8OUqjPbKOupdzXp3dZjWCGc,4403
+pydantic_core/__pycache__/__init__.cpython-310.pyc,,
+pydantic_core/__pycache__/core_schema.cpython-310.pyc,,
+pydantic_core/_pydantic_core.cpython-310-x86_64-linux-gnu.so,sha256=zm2niEcIylxNx_vxVvTeWr_FqSJUzHJxoyplEnvmaEA,4768792
+pydantic_core/_pydantic_core.pyi,sha256=xIR9CkJaClUD5HcHtGEPElBpWiD33PaxLKnss8rsuSM,43359
+pydantic_core/core_schema.py,sha256=98qpsz-jklOqmsA9h-zWg4K4jNkkk6N_nDBLW3Cjp-w,149655
+pydantic_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
diff --git a/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/WHEEL b/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/WHEEL
new file mode 100644
index 0000000000000000000000000000000000000000..1ccdf6541fd95ae8009152c2cd9ca8be003d8b0a
--- /dev/null
+++ b/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/WHEEL
@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.8.3)
+Root-Is-Purelib: false
+Tag: cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64
diff --git a/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/licenses/LICENSE b/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/licenses/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..0716871caabdbbb3e77a0371d49936cef1923ea1
--- /dev/null
+++ b/lib/python3.1/site-packages/pydantic_core-2.33.2.dist-info/licenses/LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Samuel Colvin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/lib/python3.1/site-packages/pylab.py b/lib/python3.1/site-packages/pylab.py
new file mode 100644
index 0000000000000000000000000000000000000000..64ece55b1cb2c312d8a9c2ac1fd264fa11727270
--- /dev/null
+++ b/lib/python3.1/site-packages/pylab.py
@@ -0,0 +1,3 @@
+from matplotlib.pylab import *  # noqa: F401, F403
+import matplotlib.pylab
+__doc__ = matplotlib.pylab.__doc__
diff --git a/lib/python3.1/site-packages/pymupdf/__init__.py b/lib/python3.1/site-packages/pymupdf/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae7863dbb7c182db5d6d763b94d32d8f923c7c52
--- /dev/null
+++ b/lib/python3.1/site-packages/pymupdf/__init__.py
@@ -0,0 +1,21279 @@
+'''
+PyMuPDF implemented on top of MuPDF Python bindings.
+
+License:
+
+    SPDX-License-Identifier: GPL-3.0-only
+'''
+
+# To reduce startup times, we don't import everything we require here.
+#
+import atexit
+import binascii
+import collections
+import inspect
+import io
+import math
+import os
+import pathlib
+import glob
+import re
+import string
+import sys
+import tarfile
+import time
+import typing
+import warnings
+import weakref
+import zipfile
+
+from . import extra
+
+
+# Set up g_out_log and g_out_message from environment variables.
+#
+# PYMUPDF_MESSAGE controls the destination of user messages (from function
+# `pymupdf.message()`).
+#
+# PYMUPDF_LOG controls the destination of internal development logging (from
+# function `pymupdf.log()`).
+#
+# For syntax, see _make_output()'s `text` arg.
+#
+
+def _make_output(
+        *,
+        text=None,
+        fd=None,
+        stream=None,
+        path=None,
+        path_append=None,
+        pylogging=None,
+        pylogging_logger=None,
+        pylogging_level=None,
+        pylogging_name=None,
+        default=None,
+        ):
+    '''
+    Returns a stream that writes to a specified destination, which can be a
+    file descriptor, a file, an existing stream or Python's `logging' system.
+    
+    Args:
+        text: text specification of destination.
+            fd: - write to file descriptor.
+            path: - write to file.
+            path+: - append to file.
+            logging: - write to Python `logging` module.
+                items: comma-separated  pairs.
+                    level=
+                    name=.
+                Other names are ignored.
+        
+        fd: an int file descriptor.
+        stream: something with methods .write(text) and .flush().
+            If specified we simply return .
+        path: a file path.
+            If specified we return a stream that writes to this file.
+        path_append: a file path.
+            If specified we return a stream that appends to this file.
+        pylogging*:
+            if any of these args is not None, we return a stream that writes to
+            Python's `logging` module.
+            
+            pylogging:
+                Unused other than to activate use of logging module.
+            pylogging_logger:
+                A logging.Logger; If None, set from .
+            pylogging_level:
+                An int log level, if None we use
+                pylogging_logger.getEffectiveLevel().
+            pylogging_name:
+                Only used if  is None:
+                    If  is None, we set it to 'pymupdf'.
+                    Then we do: pylogging_logger = logging.getLogger(pylogging_name)
+    '''
+    if text is not None:
+        # Textual specification, for example from from environment variable.
+        if text.startswith('fd:'):
+            fd = int(text[3:])
+        elif text.startswith('path:'):
+            path = text[5:]
+        elif text.startswith('path+'):
+            path_append = text[5:]
+        elif text.startswith('logging:'):
+            pylogging = True
+            items_d = dict()
+            items = text[8:].split(',')
+            #items_d = {n: v for (n, v) in [item.split('=', 1) for item in items]}
+            for item in items:
+                if not item:
+                    continue
+                nv = item.split('=', 1)
+                assert len(nv) == 2, f'Need `=` in {item=}.'
+                n, v = nv
+                items_d[n] = v
+            pylogging_level = items_d.get('level')
+            if pylogging_level is not None:
+                pylogging_level = int(pylogging_level)
+            pylogging_name = items_d.get('name', 'pymupdf')
+        else:
+            assert 0, f'Expected prefix `fd:`, `path:`. `path+:` or `logging:` in {text=}.'
+    
+    if fd is not None:
+        ret = open(fd, mode='w', closefd=False)
+    elif stream is not None:
+        assert hasattr(stream, 'write')
+        assert hasattr(stream, 'flush')
+        ret = stream
+    elif path is not None:
+        ret = open(path, 'w')
+    elif path_append is not None:
+        ret = open(path_append, 'a')
+    elif (0
+            or pylogging is not None
+            or pylogging_logger is not None
+            or pylogging_level is not None
+            or pylogging_name is not None
+            ):
+        import logging
+        if pylogging_logger is None:
+            if pylogging_name is None:
+                pylogging_name = 'pymupdf'
+            pylogging_logger = logging.getLogger(pylogging_name)
+        assert isinstance(pylogging_logger, logging.Logger)
+        if pylogging_level is None:
+            pylogging_level = pylogging_logger.getEffectiveLevel()
+        class Out:
+            def write(self, text):
+                # `logging` module appends newlines, but so does the `print()`
+                # functions in our caller message() and log() fns, so we need to
+                # remove them here.
+                text = text.rstrip('\n')
+                if text:
+                    pylogging_logger.log(pylogging_level, text)
+            def flush(self):
+                pass
+        ret = Out()
+    else:
+        ret = default
+    return ret
+
+# Set steam used by PyMuPDF messaging.
+_g_out_message = _make_output(text=os.environ.get('PYMUPDF_MESSAGE'), default=sys.stdout)
+
+# Set steam used by PyMuPDF development/debugging logging.
+_g_out_log = _make_output(text=os.environ.get('PYMUPDF_LOG'), default=sys.stdout)
+
+# Things for testing logging.
+_g_log_items = list()
+_g_log_items_active = False
+
+def _log_items():
+    return _g_log_items
+
+def _log_items_active(active):
+    global _g_log_items_active
+    _g_log_items_active = active
+        
+def _log_items_clear():
+    del _g_log_items[:]
+
+
+def set_messages(
+        *,
+        text=None,
+        fd=None,
+        stream=None,
+        path=None,
+        path_append=None,
+        pylogging=None,
+        pylogging_logger=None,
+        pylogging_level=None,
+        pylogging_name=None,
+        ):
+    '''
+    Sets destination of PyMuPDF messages. See _make_output() for details.
+    '''
+    global _g_out_message
+    _g_out_message = _make_output(
+            text=text,
+            fd=fd,
+            stream=stream,
+            path=path,
+            path_append=path_append,
+            pylogging=pylogging,
+            pylogging_logger=pylogging_logger,
+            pylogging_level=pylogging_level,
+            pylogging_name=pylogging_name,
+            default=_g_out_message,
+            )
+
+def set_log(
+        *,
+        text=None,
+        fd=None,
+        stream=None,
+        path=None,
+        path_append=None,
+        pylogging=None,
+        pylogging_logger=None,
+        pylogging_level=None,
+        pylogging_name=None,
+        ):
+    '''
+    Sets destination of PyMuPDF development/debugging logging. See
+    _make_output() for details.
+    '''
+    global _g_out_log
+    _g_out_log = _make_output(
+            text=text,
+            fd=fd,
+            stream=stream,
+            path=path,
+            path_append=path_append,
+            pylogging=pylogging,
+            pylogging_logger=pylogging_logger,
+            pylogging_level=pylogging_level,
+            pylogging_name=pylogging_name,
+            default=_g_out_log,
+            )
+
+def log( text='', caller=1):
+    '''
+    For development/debugging diagnostics.
+    '''
+    try:
+        stack = inspect.stack(context=0)
+    except StopIteration:
+        pass
+    else:
+        frame_record = stack[caller]
+        try:
+            filename = os.path.relpath(frame_record.filename)
+        except Exception:   # Can fail on windows.
+            filename = frame_record.filename
+        line = frame_record.lineno
+        function = frame_record.function
+        text = f'{filename}:{line}:{function}(): {text}'
+    if _g_log_items_active:
+        _g_log_items.append(text)
+    if _g_out_log:
+        print(text, file=_g_out_log, flush=1)
+
+
+def message(text=''):
+    '''
+    For user messages.
+    '''
+    # It looks like `print()` does nothing if sys.stdout is None (without
+    # raising an exception), but we don't rely on this.
+    if _g_out_message:
+        print(text, file=_g_out_message, flush=1)
+
+
+def exception_info():
+    import traceback
+    log(f'exception_info:')
+    log(traceback.format_exc())
+
+
+# PDF names must not contain these characters:
+INVALID_NAME_CHARS = set(string.whitespace + "()<>[]{}/%" + chr(0))
+
+def get_env_bool( name, default):
+    '''
+    Returns `True`, `False` or `default` depending on whether $ is '1',
+    '0' or unset. Otherwise assert-fails.
+    '''
+    v = os.environ.get( name)
+    if v is None:
+        ret = default
+    elif v == '1':
+        ret = True
+    elif v == '0':
+        ret = False
+    else:
+        assert 0, f'Unrecognised value for {name}: {v!r}'
+    if ret != default:
+        log(f'Using non-default setting from {name}: {v!r}')
+    return ret
+
+def get_env_int( name, default):
+    '''
+    Returns `True`, `False` or `default` depending on whether $ is '1',
+    '0' or unset. Otherwise assert-fails.
+    '''
+    v = os.environ.get( name)
+    if v is None:
+        ret = default
+    else:
+        ret = int(v)
+    if ret != default:
+        log(f'Using non-default setting from {name}: {v}')
+    return ret
+
+# All our `except ...` blocks output diagnostics if `g_exceptions_verbose` is
+# true.
+g_exceptions_verbose = get_env_int( 'PYMUPDF_EXCEPTIONS_VERBOSE', 1)
+
+# $PYMUPDF_USE_EXTRA overrides whether to use optimised C fns in `extra`.
+#
+g_use_extra = get_env_bool( 'PYMUPDF_USE_EXTRA', True)
+
+
+# Global switches
+#
+
+class _Globals:
+    def __init__(self):
+        self.no_device_caching = 0
+        self.small_glyph_heights = 0
+        self.subset_fontnames = 0
+        self.skip_quad_corrections = 0
+
+_globals = _Globals()
+
+
+# Optionally use MuPDF via cppyy bindings; experimental and not tested recently
+# as of 2023-01-20 11:51:40
+#
+mupdf_cppyy = os.environ.get( 'MUPDF_CPPYY')
+if mupdf_cppyy is not None:
+    # pylint: disable=all
+    log( f'{__file__}: $MUPDF_CPPYY={mupdf_cppyy!r} so attempting to import mupdf_cppyy.')
+    log( f'{__file__}: $PYTHONPATH={os.environ["PYTHONPATH"]}')
+    if mupdf_cppyy == '':
+        import mupdf_cppyy
+    else:
+        import importlib
+        mupdf_cppyy = importlib.machinery.SourceFileLoader(
+                'mupdf_cppyy',
+                mupdf_cppyy
+                ).load_module()
+    mupdf = mupdf_cppyy.cppyy.gbl.mupdf
+else:
+    # Use MuPDF Python SWIG bindings. We allow import from either our own
+    # directory for conventional wheel installs, or from separate place in case
+    # we are using a separately-installed system installation of mupdf.
+    #
+    try:
+        from . import mupdf
+    except Exception:
+        import mupdf
+    if hasattr(mupdf, 'internal_check_ndebug'):
+        mupdf.internal_check_ndebug()
+    mupdf.reinit_singlethreaded()
+
+def _int_rc(text):
+    '''
+    Converts string to int, ignoring trailing 'rc...'.
+    '''
+    rc = text.find('rc')
+    if rc >= 0:
+        text = text[:rc]
+    return int(text)
+
+# Basic version information.
+#
+pymupdf_version = "1.26.1"
+mupdf_version = mupdf.FZ_VERSION
+pymupdf_date = None
+
+# Versions as tuples; useful when comparing versions.
+#
+pymupdf_version_tuple = tuple( [_int_rc(i) for i in pymupdf_version.split('.')])
+mupdf_version_tuple = tuple( [_int_rc(i) for i in mupdf_version.split('.')])
+
+assert mupdf_version_tuple == (mupdf.FZ_VERSION_MAJOR, mupdf.FZ_VERSION_MINOR, mupdf.FZ_VERSION_PATCH), \
+        f'Inconsistent MuPDF version numbers: {mupdf_version_tuple=} != {(mupdf.FZ_VERSION_MAJOR, mupdf.FZ_VERSION_MINOR, mupdf.FZ_VERSION_PATCH)=}'
+
+# Legacy version information.
+#
+version = (pymupdf_version, mupdf_version, None)
+VersionFitz = mupdf_version
+VersionBind = pymupdf_version
+VersionDate = None
+
+
+# String formatting.
+
+def _format_g(value, *, fmt='%g'):
+    '''
+    Returns `value` formatted with mupdf.fz_format_double() if available,
+    otherwise with Python's `%`.
+
+    If `value` is a list or tuple, we return a space-separated string of
+    formatted values.
+    '''
+    if isinstance(value, (list, tuple)):
+        ret = ''
+        for v in value:
+            if ret:
+                ret += ' '
+            ret += _format_g(v, fmt=fmt)
+        return ret
+    else:
+        return mupdf.fz_format_double(fmt, value)
+        
+format_g = _format_g
+
+# ByteString is gone from typing in 3.14.
+# collections.abc.Buffer available from 3.12 only
+try:
+    ByteString = typing.ByteString
+except AttributeError:
+    ByteString = bytes | bytearray | memoryview
+
+# Names required by class method typing annotations.
+OptBytes = typing.Optional[ByteString]
+OptDict = typing.Optional[dict]
+OptFloat = typing.Optional[float]
+OptInt = typing.Union[int, None]
+OptSeq = typing.Optional[typing.Sequence]
+OptStr = typing.Optional[str]
+
+Page = 'Page_forward_decl'
+Point = 'Point_forward_decl'
+
+matrix_like = 'matrix_like'
+point_like = 'point_like'
+quad_like = 'quad_like'
+rect_like = 'rect_like'
+
+
+def _as_fz_document(document):
+    '''
+    Returns document as a mupdf.FzDocument, upcasting as required. Raises
+    'document closed' exception if closed.
+    '''
+    if isinstance(document, Document):
+        if document.is_closed:
+            raise ValueError('document closed')
+        document = document.this
+    if isinstance(document, mupdf.FzDocument):
+        return document
+    elif isinstance(document, mupdf.PdfDocument):
+        return document.super()
+    elif document is None:
+        assert 0, f'document is None'
+    else:
+        assert 0, f'Unrecognised {type(document)=}'
+
+def _as_pdf_document(document, required=True):
+    '''
+    Returns `document` downcast to a mupdf.PdfDocument. If downcast fails (i.e.
+    `document` is not actually a `PdfDocument`) then we assert-fail if `required`
+    is true (the default) else return a `mupdf.PdfDocument` with `.m_internal`
+    false.
+    '''
+    if isinstance(document, Document):
+        if document.is_closed:
+            raise ValueError('document closed')
+        document = document.this
+    if isinstance(document, mupdf.PdfDocument):
+        return document
+    elif isinstance(document, mupdf.FzDocument):
+        ret = mupdf.PdfDocument(document)
+        if required:
+            assert ret.m_internal
+        return ret
+    elif document is None:
+        assert 0, f'document is None'
+    else:
+        assert 0, f'Unrecognised {type(document)=}'
+
+def _as_fz_page(page):
+    '''
+    Returns page as a mupdf.FzPage, upcasting as required.
+    '''
+    if isinstance(page, Page):
+        page = page.this
+    if isinstance(page, mupdf.PdfPage):
+        return page.super()
+    elif isinstance(page, mupdf.FzPage):
+        return page
+    elif page is None:
+        assert 0, f'page is None'
+    else:
+        assert 0, f'Unrecognised {type(page)=}'
+
+def _as_pdf_page(page, required=True):
+    '''
+    Returns `page` downcast to a mupdf.PdfPage. If downcast fails (i.e. `page`
+    is not actually a `PdfPage`) then we assert-fail if `required` is true (the
+    default) else return a `mupdf.PdfPage` with `.m_internal` false.
+    '''
+    if isinstance(page, Page):
+        page = page.this
+    if isinstance(page, mupdf.PdfPage):
+        return page
+    elif isinstance(page, mupdf.FzPage):
+        ret = mupdf.pdf_page_from_fz_page(page)
+        if required:
+            assert ret.m_internal
+        return ret
+    elif page is None:
+        assert 0, f'page is None'
+    else:
+        assert 0, f'Unrecognised {type(page)=}'
+
+
+def _pdf_annot_page(annot):
+    '''
+    Wrapper for mupdf.pdf_annot_page() which raises an exception if 
+    is not bound to a page instead of returning a mupdf.PdfPage with
+    `.m_internal=None`.
+
+    [Some other MuPDF functions such as pdf_update_annot()` already raise a
+    similar exception if a pdf_annot's .page field is null.]
+    '''
+    page = mupdf.pdf_annot_page(annot)
+    if not page.m_internal:
+        raise RuntimeError('Annot is not bound to a page')
+    return page
+
+
+# Fixme: we don't support JM_MEMORY=1.
+JM_MEMORY = 0
+
+# Classes
+#
+
+class Annot:
+
+    def __init__(self, annot):
+        assert isinstance( annot, mupdf.PdfAnnot)
+        self.this = annot
+
+    def __repr__(self):
+        parent = getattr(self, 'parent', '<>')
+        return "'%s' annotation on %s" % (self.type[1], str(parent))
+
+    def __str__(self):
+        return self.__repr__()
+
+    def _erase(self):
+        if getattr(self, "thisown", False):
+            self.thisown = False
+
+    def _get_redact_values(self):
+        annot = self.this
+        if mupdf.pdf_annot_type(annot) != mupdf.PDF_ANNOT_REDACT:
+            return
+
+        values = dict()
+        try:
+            obj = mupdf.pdf_dict_gets(mupdf.pdf_annot_obj(annot), "RO")
+            if obj.m_internal:
+                message_warning("Ignoring redaction key '/RO'.")
+                xref = mupdf.pdf_to_num(obj)
+                values[dictkey_xref] = xref
+            obj = mupdf.pdf_dict_gets(mupdf.pdf_annot_obj(annot), "OverlayText")
+            if obj.m_internal:
+                text = mupdf.pdf_to_text_string(obj)
+                values[dictkey_text] = JM_UnicodeFromStr(text)
+            else:
+                values[dictkey_text] = ''
+            obj = mupdf.pdf_dict_get(mupdf.pdf_annot_obj(annot), PDF_NAME('Q'))
+            align = 0
+            if obj.m_internal:
+                align = mupdf.pdf_to_int(obj)
+            values[dictkey_align] = align
+        except Exception:
+            if g_exceptions_verbose:    exception_info()
+            return
+        val = values
+
+        if not val:
+            return val
+        val["rect"] = self.rect
+        text_color, fontname, fontsize = TOOLS._parse_da(self)
+        val["text_color"] = text_color
+        val["fontname"] = fontname
+        val["fontsize"] = fontsize
+        fill = self.colors["fill"]
+        val["fill"] = fill
+        return val
+
+    def _getAP(self):
+        if g_use_extra:
+            assert isinstance( self.this, mupdf.PdfAnnot)
+            ret = extra.Annot_getAP(self.this)
+            assert isinstance( ret, bytes)
+            return ret
+        else:
+            r = None
+            res = None
+            annot = self.this
+            assert isinstance( annot, mupdf.PdfAnnot)
+            annot_obj = mupdf.pdf_annot_obj( annot)
+            ap = mupdf.pdf_dict_getl( annot_obj, PDF_NAME('AP'), PDF_NAME('N'))
+            if mupdf.pdf_is_stream( ap):
+                res = mupdf.pdf_load_stream( ap)
+            if res and res.m_internal:
+                r = JM_BinFromBuffer(res)
+            return r
+
+    def _setAP(self, buffer_, rect=0):
+        try:
+            annot = self.this
+            annot_obj = mupdf.pdf_annot_obj( annot)
+            page = _pdf_annot_page(annot)
+            apobj = mupdf.pdf_dict_getl( annot_obj, PDF_NAME('AP'), PDF_NAME('N'))
+            if not apobj.m_internal:
+                raise RuntimeError( MSG_BAD_APN)
+            if not mupdf.pdf_is_stream( apobj):
+                raise RuntimeError( MSG_BAD_APN)
+            res = JM_BufferFromBytes( buffer_)
+            if not res.m_internal:
+                raise ValueError( MSG_BAD_BUFFER)
+            JM_update_stream( page.doc(), apobj, res, 1)
+            if rect:
+                bbox = mupdf.pdf_dict_get_rect( annot_obj, PDF_NAME('Rect'))
+                mupdf.pdf_dict_put_rect( apobj, PDF_NAME('BBox'), bbox)
+        except Exception:
+            if g_exceptions_verbose:    exception_info()
+
+    def _update_appearance(self, opacity=-1, blend_mode=None, fill_color=None, rotate=-1):
+        annot = self.this
+        assert annot.m_internal
+        annot_obj = mupdf.pdf_annot_obj( annot)
+        page = _pdf_annot_page(annot)
+        pdf = page.doc()
+        type_ = mupdf.pdf_annot_type( annot)
+        nfcol, fcol = JM_color_FromSequence(fill_color)
+
+        try:
+            # remove fill color from unsupported annots
+            # or if so requested
+            if nfcol == 0 or type_ not in (
+                    mupdf.PDF_ANNOT_SQUARE,
+                    mupdf.PDF_ANNOT_CIRCLE,
+                    mupdf.PDF_ANNOT_LINE,
+                    mupdf.PDF_ANNOT_POLY_LINE,
+                    mupdf.PDF_ANNOT_POLYGON
+                    ):
+                mupdf.pdf_dict_del( annot_obj, PDF_NAME('IC'))
+            elif nfcol > 0:
+                mupdf.pdf_set_annot_interior_color( annot, fcol[:nfcol])
+
+            insert_rot = 1 if rotate >= 0 else 0
+            if type_ not in (
+                    mupdf.PDF_ANNOT_CARET,
+                    mupdf.PDF_ANNOT_CIRCLE,
+                    mupdf.PDF_ANNOT_FREE_TEXT,
+                    mupdf.PDF_ANNOT_FILE_ATTACHMENT,
+                    mupdf.PDF_ANNOT_INK,
+                    mupdf.PDF_ANNOT_LINE,
+                    mupdf.PDF_ANNOT_POLY_LINE,
+                    mupdf.PDF_ANNOT_POLYGON,
+                    mupdf.PDF_ANNOT_SQUARE,
+                    mupdf.PDF_ANNOT_STAMP,
+                    mupdf.PDF_ANNOT_TEXT,
+                    ):
+                insert_rot = 0
+
+            if insert_rot:
+                mupdf.pdf_dict_put_int(annot_obj, PDF_NAME('Rotate'), rotate)
+
+            # insert fill color
+            if type_ == mupdf.PDF_ANNOT_FREE_TEXT:
+                if nfcol > 0:
+                    mupdf.pdf_set_annot_color(annot, fcol[:nfcol])
+            elif nfcol > 0:
+                col = mupdf.pdf_new_array(page.doc(), nfcol)
+                for i in range( nfcol):
+                    mupdf.pdf_array_push_real(col, fcol[i])
+                mupdf.pdf_dict_put(annot_obj, PDF_NAME('IC'), col)
+            mupdf.pdf_dirty_annot(annot)
+            mupdf.pdf_update_annot(annot) # let MuPDF update
+            pdf.resynth_required = 0
+        except Exception as e:
+            if g_exceptions_verbose:
+                exception_info()
+            message( f'cannot update annot: {e}')
+            raise
+        
+        if (opacity < 0 or opacity >= 1) and not blend_mode:    # no opacity, no blend_mode
+            return True
+
+        try:    # create or update /ExtGState
+            ap = mupdf.pdf_dict_getl(
+                    mupdf.pdf_annot_obj(annot),
+                    PDF_NAME('AP'),
+                    PDF_NAME('N')
+                    )
+            if not ap.m_internal:   # should never happen
+                raise RuntimeError( MSG_BAD_APN)
+
+            resources = mupdf.pdf_dict_get( ap, PDF_NAME('Resources'))
+            if not resources.m_internal:    # no Resources yet: make one
+                resources = mupdf.pdf_dict_put_dict( ap, PDF_NAME('Resources'), 2)
+            
+            alp0 = mupdf.pdf_new_dict( page.doc(), 3)
+            if opacity >= 0 and opacity < 1:
+                mupdf.pdf_dict_put_real( alp0, PDF_NAME('CA'), opacity)
+                mupdf.pdf_dict_put_real( alp0, PDF_NAME('ca'), opacity)
+                mupdf.pdf_dict_put_real( annot_obj, PDF_NAME('CA'), opacity)
+
+            if blend_mode:
+                mupdf.pdf_dict_put_name( alp0, PDF_NAME('BM'), blend_mode)
+                mupdf.pdf_dict_put_name( annot_obj, PDF_NAME('BM'), blend_mode)
+
+            extg = mupdf.pdf_dict_get( resources, PDF_NAME('ExtGState'))
+            if not extg.m_internal: # no ExtGState yet: make one
+                extg = mupdf.pdf_dict_put_dict( resources, PDF_NAME('ExtGState'), 2)
+
+            mupdf.pdf_dict_put( extg, PDF_NAME('H'), alp0)
+
+        except Exception as e:
+            if g_exceptions_verbose:    exception_info()
+            message( f'cannot set opacity or blend mode\n: {e}')
+            raise
+
+        return True
+
+    @property
+    def apn_bbox(self):
+        """annotation appearance bbox"""
+        CheckParent(self)
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        ap = mupdf.pdf_dict_getl(annot_obj, PDF_NAME('AP'), PDF_NAME('N'))
+        if not ap.m_internal:
+            val = JM_py_from_rect(mupdf.FzRect(mupdf.FzRect.Fixed_INFINITE))
+        else:
+            rect = mupdf.pdf_dict_get_rect(ap, PDF_NAME('BBox'))
+            val = JM_py_from_rect(rect)
+
+        val = Rect(val) * self.get_parent().transformation_matrix
+        val *= self.get_parent().derotation_matrix
+        return val
+
+    @property
+    def apn_matrix(self):
+        """annotation appearance matrix"""
+        try:
+            CheckParent(self)
+            annot = self.this
+            assert isinstance(annot, mupdf.PdfAnnot)
+            ap = mupdf.pdf_dict_getl(
+                    mupdf.pdf_annot_obj(annot),
+                    mupdf.PDF_ENUM_NAME_AP,
+                    mupdf.PDF_ENUM_NAME_N
+                    )
+            if not ap.m_internal:
+                return JM_py_from_matrix(mupdf.FzMatrix())
+            mat = mupdf.pdf_dict_get_matrix(ap, mupdf.PDF_ENUM_NAME_Matrix)
+            val = JM_py_from_matrix(mat)
+
+            val = Matrix(val)
+
+            return val
+        except Exception:
+            if g_exceptions_verbose:    exception_info()
+            raise
+
+    @property
+    def blendmode(self):
+        """annotation BlendMode"""
+        CheckParent(self)
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        obj = mupdf.pdf_dict_get(annot_obj, PDF_NAME('BM'))
+        blend_mode = None
+        if obj.m_internal:
+            blend_mode = JM_UnicodeFromStr(mupdf.pdf_to_name(obj))
+            return blend_mode
+        # loop through the /AP/N/Resources/ExtGState objects
+        obj = mupdf.pdf_dict_getl(
+                annot_obj,
+                PDF_NAME('AP'),
+                PDF_NAME('N'),
+                PDF_NAME('Resources'),
+                PDF_NAME('ExtGState'),
+                )
+        if mupdf.pdf_is_dict(obj):
+            n = mupdf.pdf_dict_len(obj)
+            for i in range(n):
+                obj1 = mupdf.pdf_dict_get_val(obj, i)
+                if mupdf.pdf_is_dict(obj1):
+                    m = mupdf.pdf_dict_len(obj1)
+                    for j in range(m):
+                        obj2 = mupdf.pdf_dict_get_key(obj1, j)
+                        if mupdf.pdf_objcmp(obj2, PDF_NAME('BM')) == 0:
+                            blend_mode = JM_UnicodeFromStr(mupdf.pdf_to_name(mupdf.pdf_dict_get_val(obj1, j)))
+                            return blend_mode
+        return blend_mode
+
+    @property
+    def border(self):
+        """Border information."""
+        CheckParent(self)
+        atype = self.type[0]
+        if atype not in (
+                mupdf.PDF_ANNOT_CIRCLE,
+                mupdf.PDF_ANNOT_FREE_TEXT,
+                mupdf.PDF_ANNOT_INK,
+                mupdf.PDF_ANNOT_LINE,
+                mupdf.PDF_ANNOT_POLY_LINE,
+                mupdf.PDF_ANNOT_POLYGON,
+                mupdf.PDF_ANNOT_SQUARE,
+                ):
+            return dict()
+        ao = mupdf.pdf_annot_obj(self.this)
+        ret = JM_annot_border(ao)
+        return ret
+
+    def clean_contents(self, sanitize=1):
+        """Clean appearance contents stream."""
+        CheckParent(self)
+        annot = self.this
+        pdf = mupdf.pdf_get_bound_document(mupdf.pdf_annot_obj(annot))
+        filter_ = _make_PdfFilterOptions(recurse=1, instance_forms=0, ascii=0, sanitize=sanitize)
+        mupdf.pdf_filter_annot_contents(pdf, annot, filter_)
+
+    @property
+    def colors(self):
+        """Color definitions."""
+        try:
+            CheckParent(self)
+            annot = self.this
+            assert isinstance(annot, mupdf.PdfAnnot)
+            return JM_annot_colors(mupdf.pdf_annot_obj(annot))
+        except Exception:
+            if g_exceptions_verbose:    exception_info()
+            raise
+
+    def delete_responses(self):
+        """Delete 'Popup' and responding annotations."""
+        CheckParent(self)
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        page = _pdf_annot_page(annot)
+        while 1:
+            irt_annot = JM_find_annot_irt(annot)
+            if not irt_annot:
+                break
+            mupdf.pdf_delete_annot(page, irt_annot)
+        mupdf.pdf_dict_del(annot_obj, PDF_NAME('Popup'))
+
+        annots = mupdf.pdf_dict_get(page.obj(), PDF_NAME('Annots'))
+        n = mupdf.pdf_array_len(annots)
+        found = 0
+        for i in range(n-1, -1, -1):
+            o = mupdf.pdf_array_get(annots, i)
+            p = mupdf.pdf_dict_get(o, PDF_NAME('Parent'))
+            if not o.m_internal:
+                continue
+            if not mupdf.pdf_objcmp(p, annot_obj):
+                mupdf.pdf_array_delete(annots, i)
+                found = 1
+        if found:
+            mupdf.pdf_dict_put(page.obj(), PDF_NAME('Annots'), annots)
+
+    @property
+    def file_info(self):
+        """Attached file information."""
+        CheckParent(self)
+        res = dict()
+        length = -1
+        size = -1
+        desc = None
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        type_ = mupdf.pdf_annot_type(annot)
+        if type_ != mupdf.PDF_ANNOT_FILE_ATTACHMENT:
+            raise TypeError( MSG_BAD_ANNOT_TYPE)
+        stream = mupdf.pdf_dict_getl(
+                annot_obj,
+                PDF_NAME('FS'),
+                PDF_NAME('EF'),
+                PDF_NAME('F'),
+                )
+        if not stream.m_internal:
+            RAISEPY( "bad PDF: file entry not found", JM_Exc_FileDataError)
+
+        fs = mupdf.pdf_dict_get(annot_obj, PDF_NAME('FS'))
+
+        o = mupdf.pdf_dict_get(fs, PDF_NAME('UF'))
+        if o.m_internal:
+            filename = mupdf.pdf_to_text_string(o)
+        else:
+            o = mupdf.pdf_dict_get(fs, PDF_NAME('F'))
+            if o.m_internal:
+                filename = mupdf.pdf_to_text_string(o)
+
+        o = mupdf.pdf_dict_get(fs, PDF_NAME('Desc'))
+        if o.m_internal:
+            desc = mupdf.pdf_to_text_string(o)
+
+        o = mupdf.pdf_dict_get(stream, PDF_NAME('Length'))
+        if o.m_internal:
+            length = mupdf.pdf_to_int(o)
+
+        o = mupdf.pdf_dict_getl(stream, PDF_NAME('Params'), PDF_NAME('Size'))
+        if o.m_internal:
+            size = mupdf.pdf_to_int(o)
+
+        res[ dictkey_filename] = JM_EscapeStrFromStr(filename)
+        res[ dictkey_descr] = JM_UnicodeFromStr(desc)
+        res[ dictkey_length] = length
+        res[ dictkey_size] = size
+        return res
+
+    @property
+    def flags(self):
+        """Flags field."""
+        CheckParent(self)
+        annot = self.this
+        return mupdf.pdf_annot_flags(annot)
+
+    def get_file(self):
+        """Retrieve attached file content."""
+        CheckParent(self)
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        type = mupdf.pdf_annot_type(annot)
+        if type != mupdf.PDF_ANNOT_FILE_ATTACHMENT:
+            raise TypeError( MSG_BAD_ANNOT_TYPE)
+        stream = mupdf.pdf_dict_getl(annot_obj, PDF_NAME('FS'), PDF_NAME('EF'), PDF_NAME('F'))
+        if not stream.m_internal:
+            RAISEPY( "bad PDF: file entry not found", JM_Exc_FileDataError)
+        buf = mupdf.pdf_load_stream(stream)
+        res = JM_BinFromBuffer(buf)
+        return res
+
+    def get_oc(self):
+        """Get annotation optional content reference."""
+        CheckParent(self)
+        oc = 0
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        obj = mupdf.pdf_dict_get(annot_obj, PDF_NAME('OC'))
+        if obj.m_internal:
+            oc = mupdf.pdf_to_num(obj)
+        return oc
+
+    # PyMuPDF doesn't seem to have this .parent member, but removing it breaks
+    # 11 tests...?
+    #@property
+    def get_parent(self):
+        try:
+            ret = getattr( self, 'parent')
+        except AttributeError:
+            page = _pdf_annot_page(self.this)
+            assert isinstance( page, mupdf.PdfPage)
+            document = Document( page.doc()) if page.m_internal else None
+            ret = Page(page, document)
+            #self.parent = weakref.proxy( ret)
+            self.parent = ret
+            #log(f'No attribute .parent: {type(self)=} {id(self)=}: have set {id(self.parent)=}.')
+            #log( f'Have set self.parent')
+        return ret
+
+    def get_pixmap(self, matrix=None, dpi=None, colorspace=None, alpha=0):
+        """annotation Pixmap"""
+
+        CheckParent(self)
+        cspaces = {"gray": csGRAY, "rgb": csRGB, "cmyk": csCMYK}
+        if type(colorspace) is str:
+            colorspace = cspaces.get(colorspace.lower(), None)
+        if dpi:
+            matrix = Matrix(dpi / 72, dpi / 72)
+        ctm = JM_matrix_from_py(matrix)
+        cs = colorspace
+        if not cs:
+            cs = mupdf.fz_device_rgb()
+
+        pix = mupdf.pdf_new_pixmap_from_annot(self.this, ctm, cs, mupdf.FzSeparations(0), alpha)
+        ret = Pixmap(pix)
+        if dpi:
+            ret.set_dpi(dpi, dpi)
+        return ret
+
+    def get_sound(self):
+        """Retrieve sound stream."""
+        CheckParent(self)
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        type = mupdf.pdf_annot_type(annot)
+        sound = mupdf.pdf_dict_get(annot_obj, PDF_NAME('Sound'))
+        if type != mupdf.PDF_ANNOT_SOUND or not sound.m_internal:
+            raise TypeError( MSG_BAD_ANNOT_TYPE)
+        if mupdf.pdf_dict_get(sound, PDF_NAME('F')).m_internal:
+            RAISEPY( "unsupported sound stream", JM_Exc_FileDataError)
+        res = dict()
+        obj = mupdf.pdf_dict_get(sound, PDF_NAME('R'))
+        if obj.m_internal:
+            res['rate'] = mupdf.pdf_to_real(obj)
+        obj = mupdf.pdf_dict_get(sound, PDF_NAME('C'))
+        if obj.m_internal:
+            res['channels'] = mupdf.pdf_to_int(obj)
+        obj = mupdf.pdf_dict_get(sound, PDF_NAME('B'))
+        if obj.m_internal:
+            res['bps'] = mupdf.pdf_to_int(obj)
+        obj = mupdf.pdf_dict_get(sound, PDF_NAME('E'))
+        if obj.m_internal:
+            res['encoding'] = mupdf.pdf_to_name(obj)
+        obj = mupdf.pdf_dict_gets(sound, "CO")
+        if obj.m_internal:
+            res['compression'] = mupdf.pdf_to_name(obj)
+        buf = mupdf.pdf_load_stream(sound)
+        stream = JM_BinFromBuffer(buf)
+        res['stream'] = stream
+        return res
+
+    def get_textpage(self, clip=None, flags=0):
+        """Make annotation TextPage."""
+        CheckParent(self)
+        options = mupdf.FzStextOptions(flags)
+        if clip:
+            assert hasattr(mupdf, 'FZ_STEXT_CLIP_RECT'), f'MuPDF-{mupdf_version} does not support FZ_STEXT_CLIP_RECT.'
+            clip2 = JM_rect_from_py(clip)
+            options.clip = clip2.internal()
+            options.flags |= mupdf.FZ_STEXT_CLIP_RECT
+        annot = self.this
+        stextpage = mupdf.FzStextPage(annot, options)
+        ret = TextPage(stextpage)
+        p = self.get_parent()
+        if isinstance(p, weakref.ProxyType):
+            ret.parent = p
+        else:
+            ret.parent = weakref.proxy(p)
+        return ret
+
+    @property
+    def has_popup(self):
+        """Check if annotation has a Popup."""
+        CheckParent(self)
+        annot = self.this
+        obj = mupdf.pdf_dict_get(mupdf.pdf_annot_obj(annot), PDF_NAME('Popup'))
+        return True if obj.m_internal else False
+
+    @property
+    def info(self):
+        """Various information details."""
+        CheckParent(self)
+        annot = self.this
+        res = dict()
+
+        res[dictkey_content] = JM_UnicodeFromStr(mupdf.pdf_annot_contents(annot))
+
+        o = mupdf.pdf_dict_get(mupdf.pdf_annot_obj(annot), PDF_NAME('Name'))
+        res[dictkey_name] = JM_UnicodeFromStr(mupdf.pdf_to_name(o))
+
+        # Title (= author)
+        o = mupdf.pdf_dict_get(mupdf.pdf_annot_obj(annot), PDF_NAME('T'))
+        res[dictkey_title] = JM_UnicodeFromStr(mupdf.pdf_to_text_string(o))
+
+        # CreationDate
+        o = mupdf.pdf_dict_gets(mupdf.pdf_annot_obj(annot), "CreationDate")
+        res[dictkey_creationDate] = JM_UnicodeFromStr(mupdf.pdf_to_text_string(o))
+
+        # ModDate
+        o = mupdf.pdf_dict_get(mupdf.pdf_annot_obj(annot), PDF_NAME('M'))
+        res[dictkey_modDate] = JM_UnicodeFromStr(mupdf.pdf_to_text_string(o))
+
+        # Subj
+        o = mupdf.pdf_dict_gets(mupdf.pdf_annot_obj(annot), "Subj")
+        res[dictkey_subject] = mupdf.pdf_to_text_string(o)
+
+        # Identification (PDF key /NM)
+        o = mupdf.pdf_dict_gets(mupdf.pdf_annot_obj(annot), "NM")
+        res[dictkey_id] = JM_UnicodeFromStr(mupdf.pdf_to_text_string(o))
+
+        return res
+
+    @property
+    def irt_xref(self):
+        '''
+        annotation IRT xref
+        '''
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj( annot)
+        irt = mupdf.pdf_dict_get( annot_obj, PDF_NAME('IRT'))
+        if not irt.m_internal:
+            return 0
+        return mupdf.pdf_to_num( irt)
+
+    @property
+    def is_open(self):
+        """Get 'open' status of annotation or its Popup."""
+        CheckParent(self)
+        return mupdf.pdf_annot_is_open(self.this)
+
+    @property
+    def language(self):
+        """annotation language"""
+        this_annot = self.this
+        lang = mupdf.pdf_annot_language(this_annot)
+        if lang == mupdf.FZ_LANG_UNSET:
+            return
+        assert hasattr(mupdf, 'fz_string_from_text_language2')
+        return mupdf.fz_string_from_text_language2(lang)
+
+    @property
+    def line_ends(self):
+        """Line end codes."""
+        CheckParent(self)
+        annot = self.this
+        # return nothing for invalid annot types
+        if not mupdf.pdf_annot_has_line_ending_styles(annot):
+            return
+        lstart = mupdf.pdf_annot_line_start_style(annot)
+        lend = mupdf.pdf_annot_line_end_style(annot)
+        return lstart, lend
+
+    @property
+    def next(self):
+        """Next annotation."""
+        CheckParent(self)
+        this_annot = self.this
+        assert isinstance(this_annot, mupdf.PdfAnnot)
+        assert this_annot.m_internal
+        type_ = mupdf.pdf_annot_type(this_annot)
+        if type_ != mupdf.PDF_ANNOT_WIDGET:
+            annot = mupdf.pdf_next_annot(this_annot)
+        else:
+            annot = mupdf.pdf_next_widget(this_annot)
+
+        val = Annot(annot) if annot.m_internal else None
+        if not val:
+            return None
+        val.thisown = True
+        assert val.get_parent().this.m_internal_value() == self.get_parent().this.m_internal_value()
+        val.parent._annot_refs[id(val)] = val
+
+        if val.type[0] == mupdf.PDF_ANNOT_WIDGET:
+            widget = Widget()
+            TOOLS._fill_widget(val, widget)
+            val = widget
+        return val
+
+    @property
+    def opacity(self):
+        """Opacity."""
+        CheckParent(self)
+        annot = self.this
+        opy = -1
+        ca = mupdf.pdf_dict_get( mupdf.pdf_annot_obj(annot), mupdf.PDF_ENUM_NAME_CA)
+        if mupdf.pdf_is_number(ca):
+            opy = mupdf.pdf_to_real(ca)
+        return opy
+
+    @property
+    def popup_rect(self):
+        """annotation 'Popup' rectangle"""
+        CheckParent(self)
+        rect = mupdf.FzRect(mupdf.FzRect.Fixed_INFINITE)
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj( annot)
+        obj = mupdf.pdf_dict_get( annot_obj, PDF_NAME('Popup'))
+        if obj.m_internal:
+            rect = mupdf.pdf_dict_get_rect(obj, PDF_NAME('Rect'))
+            #log( '{rect=}')
+        val = JM_py_from_rect(rect)
+        #log( '{val=}')
+        
+        val = Rect(val) * self.get_parent().transformation_matrix
+        val *= self.get_parent().derotation_matrix
+        
+        return val
+
+    @property
+    def popup_xref(self):
+        """annotation 'Popup' xref"""
+        CheckParent(self)
+        xref = 0
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        obj = mupdf.pdf_dict_get(annot_obj, PDF_NAME('Popup'))
+        if obj.m_internal:
+            xref = mupdf.pdf_to_num(obj)
+        return xref
+
+    @property
+    def rect(self):
+        """annotation rectangle"""
+        if g_use_extra:
+            val = extra.Annot_rect3( self.this)
+        else:
+            val = mupdf.pdf_bound_annot(self.this)
+        val = Rect(val)
+        
+        # Caching self.parent_() reduces 1000x from 0.07 to 0.04.
+        #
+        p = self.get_parent()
+        #p = getattr( self, 'parent', None)
+        #if p is None:
+        #    p = self.parent
+        #    self.parent = p
+        #p = self.parent_()
+        val *= p.derotation_matrix
+        return val
+
+    @property
+    def rect_delta(self):
+        '''
+        annotation delta values to rectangle
+        '''
+        annot_obj = mupdf.pdf_annot_obj(self.this)
+        arr = mupdf.pdf_dict_get( annot_obj, PDF_NAME('RD'))
+        if mupdf.pdf_array_len( arr) == 4:
+            return (
+                    mupdf.pdf_to_real( mupdf.pdf_array_get( arr, 0)),
+                    mupdf.pdf_to_real( mupdf.pdf_array_get( arr, 1)),
+                    -mupdf.pdf_to_real( mupdf.pdf_array_get( arr, 2)),
+                    -mupdf.pdf_to_real( mupdf.pdf_array_get( arr, 3)),
+                    )
+
+    @property
+    def rotation(self):
+        """annotation rotation"""
+        CheckParent(self)
+        annot = self.this
+        rotation = mupdf.pdf_dict_get( mupdf.pdf_annot_obj(annot), mupdf.PDF_ENUM_NAME_Rotate)
+        if not rotation.m_internal:
+            return -1
+        return mupdf.pdf_to_int( rotation)
+
+    def set_apn_bbox(self, bbox):
+        """
+        Set annotation appearance bbox.
+        """
+        CheckParent(self)
+        page = self.get_parent()
+        rot = page.rotation_matrix
+        mat = page.transformation_matrix
+        bbox *= rot * ~mat
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        ap = mupdf.pdf_dict_getl(annot_obj, PDF_NAME('AP'), PDF_NAME('N'))
+        if not ap.m_internal:
+            raise RuntimeError( MSG_BAD_APN)
+        rect = JM_rect_from_py(bbox)
+        mupdf.pdf_dict_put_rect(ap, PDF_NAME('BBox'), rect)
+
+    def set_apn_matrix(self, matrix):
+        """Set annotation appearance matrix."""
+        CheckParent(self)
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        ap = mupdf.pdf_dict_getl(annot_obj, PDF_NAME('AP'), PDF_NAME('N'))
+        if not ap.m_internal:
+            raise RuntimeError( MSG_BAD_APN)
+        mat = JM_matrix_from_py(matrix)
+        mupdf.pdf_dict_put_matrix(ap, PDF_NAME('Matrix'), mat)
+
+    def set_blendmode(self, blend_mode):
+        """Set annotation BlendMode."""
+        CheckParent(self)
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        mupdf.pdf_dict_put_name(annot_obj, PDF_NAME('BM'), blend_mode)
+
+    def set_border(self, border=None, width=-1, style=None, dashes=None, clouds=-1):
+        """Set border properties.
+
+        Either a dict, or direct arguments width, style, dashes or clouds."""
+        CheckParent(self)
+        atype, atname = self.type[:2]  # annotation type
+        if atype not in (
+                mupdf.PDF_ANNOT_CIRCLE,
+                mupdf.PDF_ANNOT_FREE_TEXT,
+                mupdf.PDF_ANNOT_INK,
+                mupdf.PDF_ANNOT_LINE,
+                mupdf.PDF_ANNOT_POLY_LINE,
+                mupdf.PDF_ANNOT_POLYGON,
+                mupdf.PDF_ANNOT_SQUARE,
+                ):
+            message(f"Cannot set border for '{atname}'.")
+            return None
+        if atype not in (
+                mupdf.PDF_ANNOT_CIRCLE,
+                mupdf.PDF_ANNOT_FREE_TEXT,
+                mupdf.PDF_ANNOT_POLYGON,
+                mupdf.PDF_ANNOT_SQUARE,
+                ):
+            if clouds > 0:
+                message(f"Cannot set cloudy border for '{atname}'.")
+                clouds = -1  # do not set border effect
+        if type(border) is not dict:
+            border = {"width": width, "style": style, "dashes": dashes, "clouds": clouds}
+        border.setdefault("width", -1)
+        border.setdefault("style", None)
+        border.setdefault("dashes", None)
+        border.setdefault("clouds", -1)
+        if border["width"] is None:
+            border["width"] = -1
+        if border["clouds"] is None:
+            border["clouds"] = -1
+        if hasattr(border["dashes"], "__getitem__"):  # ensure sequence items are integers
+            border["dashes"] = tuple(border["dashes"])
+            for item in border["dashes"]:
+                if not isinstance(item, int):
+                    border["dashes"] = None
+                    break
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj( annot)
+        pdf = mupdf.pdf_get_bound_document( annot_obj)
+        return JM_annot_set_border( border, pdf, annot_obj)
+
+    def set_colors(self, colors=None, stroke=None, fill=None):
+        """Set 'stroke' and 'fill' colors.
+
+        Use either a dict or the direct arguments.
+        """
+        if self.type[0] == mupdf.PDF_ANNOT_FREE_TEXT:
+            raise ValueError("cannot be used for FreeText annotations")
+
+        CheckParent(self)
+        doc = self.get_parent().parent
+        if type(colors) is not dict:
+            colors = {"fill": fill, "stroke": stroke}
+        fill = colors.get("fill")
+        stroke = colors.get("stroke")
+
+        fill_annots = (mupdf.PDF_ANNOT_CIRCLE, mupdf.PDF_ANNOT_SQUARE, mupdf.PDF_ANNOT_LINE, mupdf.PDF_ANNOT_POLY_LINE, mupdf.PDF_ANNOT_POLYGON,
+                       mupdf.PDF_ANNOT_REDACT,)
+
+        if stroke in ([], ()):
+            doc.xref_set_key(self.xref, "C", "[]")
+        elif stroke is not None:
+            if hasattr(stroke, "__float__"):
+                stroke = [float(stroke)]
+            CheckColor(stroke)
+            assert len(stroke) in (1, 3, 4)
+            s = f"[{_format_g(stroke)}]"
+            doc.xref_set_key(self.xref, "C", s)
+
+        if fill and self.type[0] not in fill_annots:
+            message("Warning: fill color ignored for annot type '%s'." % self.type[1])
+            return
+        if fill in ([], ()):
+            doc.xref_set_key(self.xref, "IC", "[]")
+        elif fill is not None:
+            if hasattr(fill, "__float__"):
+                fill = [float(fill)]
+            CheckColor(fill)
+            assert len(fill) in (1, 3, 4)
+            s = f"[{_format_g(fill)}]"
+            doc.xref_set_key(self.xref, "IC", s)
+
+    def set_flags(self, flags):
+        """Set annotation flags."""
+        CheckParent(self)
+        annot = self.this
+        mupdf.pdf_set_annot_flags(annot, flags)
+
+    def set_info(self, info=None, content=None, title=None, creationDate=None, modDate=None, subject=None):
+        """Set various properties."""
+        CheckParent(self)
+        if type(info) is dict:  # build the args from the dictionary
+            content = info.get("content", None)
+            title = info.get("title", None)
+            creationDate = info.get("creationDate", None)
+            modDate = info.get("modDate", None)
+            subject = info.get("subject", None)
+            info = None
+        annot = self.this
+        # use this to indicate a 'markup' annot type
+        is_markup = mupdf.pdf_annot_has_author(annot)
+        # contents
+        if content:
+            mupdf.pdf_set_annot_contents(annot, content)
+        if is_markup:
+            # title (= author)
+            if title:
+                mupdf.pdf_set_annot_author(annot, title)
+            # creation date
+            if creationDate:
+                mupdf.pdf_dict_put_text_string(mupdf.pdf_annot_obj(annot), PDF_NAME('CreationDate'), creationDate)
+            # mod date
+            if modDate:
+                mupdf.pdf_dict_put_text_string(mupdf.pdf_annot_obj(annot), PDF_NAME('M'), modDate)
+            # subject
+            if subject:
+                mupdf.pdf_dict_puts(mupdf.pdf_annot_obj(annot), "Subj", mupdf.pdf_new_text_string(subject))
+
+    def set_irt_xref(self, xref):
+        '''
+        Set annotation IRT xref
+        '''
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj( annot)
+        page = _pdf_annot_page(annot)
+        if xref < 1 or xref >= mupdf.pdf_xref_len( page.doc()):
+            raise ValueError( MSG_BAD_XREF)
+        irt = mupdf.pdf_new_indirect( page.doc(), xref, 0)
+        subt = mupdf.pdf_dict_get( irt, PDF_NAME('Subtype'))
+        irt_subt = mupdf.pdf_annot_type_from_string( mupdf.pdf_to_name( subt))
+        if irt_subt < 0:
+            raise ValueError( MSG_IS_NO_ANNOT)
+        mupdf.pdf_dict_put( annot_obj, PDF_NAME('IRT'), irt)
+
+    def set_language(self, language=None):
+        """Set annotation language."""
+        CheckParent(self)
+        this_annot = self.this
+        if not language:
+            lang = mupdf.FZ_LANG_UNSET
+        else:
+            lang = mupdf.fz_text_language_from_string(language)
+        mupdf.pdf_set_annot_language(this_annot, lang)
+
+    def set_line_ends(self, start, end):
+        """Set line end codes."""
+        CheckParent(self)
+        annot = self.this
+        if mupdf.pdf_annot_has_line_ending_styles(annot):
+            mupdf.pdf_set_annot_line_ending_styles(annot, start, end)
+        else:
+            message_warning("bad annot type for line ends")
+
+    def set_name(self, name):
+        """Set /Name (icon) of annotation."""
+        CheckParent(self)
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        mupdf.pdf_dict_put_name(annot_obj, PDF_NAME('Name'), name)
+
+    def set_oc(self, oc=0):
+        """Set / remove annotation OC xref."""
+        CheckParent(self)
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        if not oc:
+            mupdf.pdf_dict_del(annot_obj, PDF_NAME('OC'))
+        else:
+            JM_add_oc_object(mupdf.pdf_get_bound_document(annot_obj), annot_obj, oc)
+
+    def set_opacity(self, opacity):
+        """Set opacity."""
+        CheckParent(self)
+        annot = self.this
+        if not _INRANGE(opacity, 0.0, 1.0):
+            mupdf.pdf_set_annot_opacity(annot, 1)
+            return
+        mupdf.pdf_set_annot_opacity(annot, opacity)
+        if opacity < 1.0:
+            page = _pdf_annot_page(annot)
+            page.transparency = 1
+
+    def set_open(self, is_open):
+        """Set 'open' status of annotation or its Popup."""
+        CheckParent(self)
+        annot = self.this
+        mupdf.pdf_set_annot_is_open(annot, is_open)
+
+    def set_popup(self, rect):
+        '''
+        Create annotation 'Popup' or update rectangle.
+        '''
+        CheckParent(self)
+        annot = self.this
+        pdfpage = _pdf_annot_page(annot)
+        rot = JM_rotate_page_matrix(pdfpage)
+        r = mupdf.fz_transform_rect(JM_rect_from_py(rect), rot)
+        mupdf.pdf_set_annot_popup(annot, r)
+
+    def set_rect(self, rect):
+        """Set annotation rectangle."""
+        CheckParent(self)
+        annot = self.this
+        
+        pdfpage = _pdf_annot_page(annot)
+        rot = JM_rotate_page_matrix(pdfpage)
+        r = mupdf.fz_transform_rect(JM_rect_from_py(rect), rot)
+        if mupdf.fz_is_empty_rect(r) or mupdf.fz_is_infinite_rect(r):
+            raise ValueError( MSG_BAD_RECT)
+        try:
+            mupdf.pdf_set_annot_rect(annot, r)
+        except Exception as e:
+            message(f'cannot set rect: {e}')
+            return False
+
+    def set_rotation(self, rotate=0):
+        """Set annotation rotation."""
+        CheckParent(self)
+        
+        annot = self.this
+        type = mupdf.pdf_annot_type(annot)
+        if type not in (
+                mupdf.PDF_ANNOT_CARET,
+                mupdf.PDF_ANNOT_CIRCLE,
+                mupdf.PDF_ANNOT_FREE_TEXT,
+                mupdf.PDF_ANNOT_FILE_ATTACHMENT,
+                mupdf.PDF_ANNOT_INK,
+                mupdf.PDF_ANNOT_LINE,
+                mupdf.PDF_ANNOT_POLY_LINE,
+                mupdf.PDF_ANNOT_POLYGON,
+                mupdf.PDF_ANNOT_SQUARE,
+                mupdf.PDF_ANNOT_STAMP,
+                mupdf.PDF_ANNOT_TEXT,
+                ):
+            return
+        rot = rotate
+        while rot < 0:
+            rot += 360
+        while rot >= 360:
+            rot -= 360
+        if type == mupdf.PDF_ANNOT_FREE_TEXT and rot % 90 != 0:
+            rot = 0
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        mupdf.pdf_dict_put_int(annot_obj, PDF_NAME('Rotate'), rot)
+
+    @property
+    def type(self):
+        """annotation type"""
+        CheckParent(self)
+        if not self.this.m_internal:
+            return 'null'
+        type_ = mupdf.pdf_annot_type(self.this)
+        c = mupdf.pdf_string_from_annot_type(type_)
+        o = mupdf.pdf_dict_gets( mupdf.pdf_annot_obj(self.this), 'IT')
+        if not o.m_internal or mupdf.pdf_is_name(o):
+            return (type_, c)
+        it = mupdf.pdf_to_name(o)
+        return (type_, c, it)
+
+    def update(self,
+            blend_mode: OptStr =None,
+            opacity: OptFloat =None,
+            fontsize: float =0,
+            fontname: OptStr =None,
+            text_color: OptSeq =None,
+            border_color: OptSeq =None,
+            fill_color: OptSeq =None,
+            cross_out: bool =True,
+            rotate: int =-1,
+            ):
+        """Update annot appearance.
+
+        Notes:
+            Depending on the annot type, some parameters make no sense,
+            while others are only available in this method to achieve the
+            desired result. This is especially true for 'FreeText' annots.
+        Args:
+            blend_mode: set the blend mode, all annotations.
+            opacity: set the opacity, all annotations.
+            fontsize: set fontsize, 'FreeText' only.
+            fontname: set the font, 'FreeText' only.
+            border_color: set border color, 'FreeText' only.
+            text_color: set text color, 'FreeText' only.
+            fill_color: set fill color, all annotations.
+            cross_out: draw diagonal lines, 'Redact' only.
+            rotate: set rotation, 'FreeText' and some others.
+        """
+        annot_obj = mupdf.pdf_annot_obj(self.this)
+        
+        if border_color:
+            is_rich_text = mupdf.pdf_dict_get(annot_obj, PDF_NAME("RC"))
+            if not is_rich_text:
+                raise ValueError("cannot set border_color if rich_text is False")
+        Annot.update_timing_test()
+        CheckParent(self)
+        def color_string(cs, code):
+            """Return valid PDF color operator for a given color sequence.
+            """
+            cc = ColorCode(cs, code)
+            if not cc:
+                return b""
+            return (cc + "\n").encode()
+
+        annot_type = self.type[0]  # get the annot type
+
+        dt = self.border.get("dashes", None)  # get the dashes spec
+        bwidth = self.border.get("width", -1)  # get border line width
+        stroke = self.colors["stroke"]  # get the stroke color
+        if fill_color is not None:
+            fill = fill_color
+        else:
+            fill = self.colors["fill"]
+        rect = None  # self.rect  # prevent MuPDF fiddling with it
+        apnmat = self.apn_matrix  # prevent MuPDF fiddling with it
+        if rotate != -1:  # sanitize rotation value
+            while rotate < 0:
+                rotate += 360
+            while rotate >= 360:
+                rotate -= 360
+            if annot_type == mupdf.PDF_ANNOT_FREE_TEXT and rotate % 90 != 0:
+                rotate = 0
+
+        #------------------------------------------------------------------
+        # handle opacity and blend mode
+        #------------------------------------------------------------------
+        if blend_mode is None:
+            blend_mode = self.blendmode
+        if not hasattr(opacity, "__float__"):
+            opacity = self.opacity
+
+        if 0 <= opacity < 1 or blend_mode:
+            opa_code = "/H gs\n"  # then we must reference this 'gs'
+        else:
+            opa_code = ""
+
+        if annot_type == mupdf.PDF_ANNOT_FREE_TEXT:
+            CheckColor(text_color)
+            CheckColor(fill_color)
+            tcol, fname, fsize = TOOLS._parse_da(self)
+
+            # read and update default appearance as necessary
+            if fsize <= 0:
+                fsize = 12
+            if text_color:
+                tcol = text_color
+            if fontname:
+                fname = fontname
+            if fontsize > 0:
+                fsize = fontsize
+            JM_make_annot_DA(self, len(tcol), tcol, fname, fsize)
+            blend_mode = None  # not supported for free text annotations!
+
+        #------------------------------------------------------------------
+        # now invoke MuPDF to update the annot appearance
+        #------------------------------------------------------------------
+        val = self._update_appearance(
+            opacity=opacity,
+            blend_mode=blend_mode,
+            fill_color=fill,
+            rotate=rotate,
+        )
+        if val is False:
+            raise RuntimeError("Error updating annotation.")
+
+        if annot_type == mupdf.PDF_ANNOT_FREE_TEXT:
+            # in absence of previous opacity, we may need to modify the AP
+            ap = self._getAP()
+            if 0 <= opacity < 1 and not ap.startswith(b"/H gs"):
+                self._setAP(b"/H gs\n" + ap)
+            return
+
+        bfill = color_string(fill, "f")
+        bstroke = color_string(stroke, "c")
+
+        p_ctm = self.get_parent().transformation_matrix
+        imat = ~p_ctm  # inverse page transf. matrix
+
+        if dt:
+            dashes = "[" + " ".join(map(str, dt)) + "] 0 d\n"
+            dashes = dashes.encode("utf-8")
+        else:
+            dashes = None
+
+        if self.line_ends:
+            line_end_le, line_end_ri = self.line_ends
+        else:
+            line_end_le, line_end_ri = 0, 0  # init line end codes
+
+        # read contents as created by MuPDF
+        ap = self._getAP()
+        ap_tab = ap.splitlines()  # split in single lines
+        ap_updated = False  # assume we did nothing
+
+        if annot_type == mupdf.PDF_ANNOT_REDACT:
+            if cross_out:  # create crossed-out rect
+                ap_updated = True
+                ap_tab = ap_tab[:-1]
+                _, LL, LR, UR, UL = ap_tab
+                ap_tab.append(LR)
+                ap_tab.append(LL)
+                ap_tab.append(UR)
+                ap_tab.append(LL)
+                ap_tab.append(UL)
+                ap_tab.append(b"S")
+
+            if bwidth > 0 or bstroke != b"":
+                ap_updated = True
+                ntab = [_format_g(bwidth).encode() + b" w"] if bwidth > 0 else []
+                for line in ap_tab:
+                    if line.endswith(b"w"):
+                        continue
+                    if line.endswith(b"RG") and bstroke != b"":
+                        line = bstroke[:-1]
+                    ntab.append(line)
+                ap_tab = ntab
+
+            ap = b"\n".join(ap_tab)
+
+        if annot_type in (mupdf.PDF_ANNOT_POLYGON, mupdf.PDF_ANNOT_POLY_LINE):
+            ap = b"\n".join(ap_tab[:-1]) + b"\n"
+            ap_updated = True
+            if bfill != b"":
+                if annot_type == mupdf.PDF_ANNOT_POLYGON:
+                    ap = ap + bfill + b"b"  # close, fill, and stroke
+                elif annot_type == mupdf.PDF_ANNOT_POLY_LINE:
+                    ap = ap + b"S"  # stroke
+            else:
+                if annot_type == mupdf.PDF_ANNOT_POLYGON:
+                    ap = ap + b"s"  # close and stroke
+                elif annot_type == mupdf.PDF_ANNOT_POLY_LINE:
+                    ap = ap + b"S"  # stroke
+
+        if dashes is not None:  # handle dashes
+            ap = dashes + ap
+            # reset dashing - only applies for LINE annots with line ends given
+            ap = ap.replace(b"\nS\n", b"\nS\n[] 0 d\n", 1)
+            ap_updated = True
+
+        if opa_code:
+            ap = opa_code.encode("utf-8") + ap
+            ap_updated = True
+
+        ap = b"q\n" + ap + b"\nQ\n"
+        #----------------------------------------------------------------------
+        # the following handles line end symbols for 'Polygon' and 'Polyline'
+        #----------------------------------------------------------------------
+        if line_end_le + line_end_ri > 0 and annot_type in (mupdf.PDF_ANNOT_POLYGON, mupdf.PDF_ANNOT_POLY_LINE):
+
+            le_funcs = (None, TOOLS._le_square, TOOLS._le_circle,
+                        TOOLS._le_diamond, TOOLS._le_openarrow,
+                        TOOLS._le_closedarrow, TOOLS._le_butt,
+                        TOOLS._le_ropenarrow, TOOLS._le_rclosedarrow,
+                        TOOLS._le_slash)
+            le_funcs_range = range(1, len(le_funcs))
+            d = 2 * max(1, self.border["width"])
+            rect = self.rect + (-d, -d, d, d)
+            ap_updated = True
+            points = self.vertices
+            if line_end_le in le_funcs_range:
+                p1 = Point(points[0]) * imat
+                p2 = Point(points[1]) * imat
+                left = le_funcs[line_end_le](self, p1, p2, False, fill_color)
+                ap += left.encode()
+            if line_end_ri in le_funcs_range:
+                p1 = Point(points[-2]) * imat
+                p2 = Point(points[-1]) * imat
+                left = le_funcs[line_end_ri](self, p1, p2, True, fill_color)
+                ap += left.encode()
+
+        if ap_updated:
+            if rect:                        # rect modified here?
+                self.set_rect(rect)
+                self._setAP(ap, rect=1)
+            else:
+                self._setAP(ap, rect=0)
+
+        #-------------------------------
+        # handle annotation rotations
+        #-------------------------------
+        if annot_type not in (  # only these types are supported
+                mupdf.PDF_ANNOT_CARET,
+                mupdf.PDF_ANNOT_CIRCLE,
+                mupdf.PDF_ANNOT_FILE_ATTACHMENT,
+                mupdf.PDF_ANNOT_INK,
+                mupdf.PDF_ANNOT_LINE,
+                mupdf.PDF_ANNOT_POLY_LINE,
+                mupdf.PDF_ANNOT_POLYGON,
+                mupdf.PDF_ANNOT_SQUARE,
+                mupdf.PDF_ANNOT_STAMP,
+                mupdf.PDF_ANNOT_TEXT,
+                ):
+            return
+
+        rot = self.rotation  # get value from annot object
+        if rot == -1:  # nothing to change
+            return
+
+        M = (self.rect.tl + self.rect.br) / 2  # center of annot rect
+
+        if rot == 0:  # undo rotations
+            if abs(apnmat - Matrix(1, 1)) < 1e-5:
+                return  # matrix already is a no-op
+            quad = self.rect.morph(M, ~apnmat)  # derotate rect
+            self.setRect(quad.rect)
+            self.set_apn_matrix(Matrix(1, 1))  # appearance matrix = no-op
+            return
+
+        mat = Matrix(rot)
+        quad = self.rect.morph(M, mat)
+        self.set_rect(quad.rect)
+        self.set_apn_matrix(apnmat * mat)
+
+    def update_file(self, buffer_=None, filename=None, ufilename=None, desc=None):
+        """Update attached file."""
+        CheckParent(self)
+        annot = self.this
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        pdf = mupdf.pdf_get_bound_document(annot_obj)  # the owning PDF
+        type = mupdf.pdf_annot_type(annot)
+        if type != mupdf.PDF_ANNOT_FILE_ATTACHMENT:
+            raise TypeError( MSG_BAD_ANNOT_TYPE)
+        stream = mupdf.pdf_dict_getl(annot_obj, PDF_NAME('FS'), PDF_NAME('EF'), PDF_NAME('F'))
+        # the object for file content
+        if not stream.m_internal:
+            RAISEPY( "bad PDF: no /EF object", JM_Exc_FileDataError)
+
+        fs = mupdf.pdf_dict_get(annot_obj, PDF_NAME('FS'))
+
+        # file content given
+        res = JM_BufferFromBytes(buffer_)
+        if buffer_ and not res.m_internal:
+            raise ValueError( MSG_BAD_BUFFER)
+        if res:
+            JM_update_stream(pdf, stream, res, 1)
+            # adjust /DL and /Size parameters
+            len, _ = mupdf.fz_buffer_storage(res)
+            l = mupdf.pdf_new_int(len)
+            mupdf.pdf_dict_put(stream, PDF_NAME('DL'), l)
+            mupdf.pdf_dict_putl(stream, l, PDF_NAME('Params'), PDF_NAME('Size'))
+
+        if filename:
+            mupdf.pdf_dict_put_text_string(stream, PDF_NAME('F'), filename)
+            mupdf.pdf_dict_put_text_string(fs, PDF_NAME('F'), filename)
+            mupdf.pdf_dict_put_text_string(stream, PDF_NAME('UF'), filename)
+            mupdf.pdf_dict_put_text_string(fs, PDF_NAME('UF'), filename)
+            mupdf.pdf_dict_put_text_string(annot_obj, PDF_NAME('Contents'), filename)
+
+        if ufilename:
+            mupdf.pdf_dict_put_text_string(stream, PDF_NAME('UF'), ufilename)
+            mupdf.pdf_dict_put_text_string(fs, PDF_NAME('UF'), ufilename)
+
+        if desc:
+            mupdf.pdf_dict_put_text_string(stream, PDF_NAME('Desc'), desc)
+            mupdf.pdf_dict_put_text_string(fs, PDF_NAME('Desc'), desc)
+
+    @staticmethod
+    def update_timing_test():
+        total = 0
+        for i in range( 30*1000):
+            total += i
+        return total
+    
+    @property
+    def vertices(self):
+        """annotation vertex points"""
+        CheckParent(self)
+        annot = self.this
+        assert isinstance(annot, mupdf.PdfAnnot)
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        page = _pdf_annot_page(annot)
+        page_ctm = mupdf.FzMatrix()   # page transformation matrix
+        dummy = mupdf.FzRect()  # Out-param for mupdf.pdf_page_transform().
+        mupdf.pdf_page_transform(page, dummy, page_ctm)
+        derot = JM_derotate_page_matrix(page)
+        page_ctm = mupdf.fz_concat(page_ctm, derot)
+
+        #----------------------------------------------------------------
+        # The following objects occur in different annotation types.
+        # So we are sure that (!o) occurs at most once.
+        # Every pair of floats is one point, that needs to be separately
+        # transformed with the page transformation matrix.
+        #----------------------------------------------------------------
+        o = mupdf.pdf_dict_get(annot_obj, PDF_NAME('Vertices'))
+        if not o.m_internal:    o = mupdf.pdf_dict_get(annot_obj, PDF_NAME('L'))
+        if not o.m_internal:    o = mupdf.pdf_dict_get(annot_obj, PDF_NAME('QuadPoints'))
+        if not o.m_internal:    o = mupdf.pdf_dict_gets(annot_obj, 'CL')
+        
+        if o.m_internal:
+            # handle lists with 1-level depth
+            # weiter
+            res = []
+            for i in range(0, mupdf.pdf_array_len(o), 2):
+                x = mupdf.pdf_to_real(mupdf.pdf_array_get(o, i))
+                y = mupdf.pdf_to_real(mupdf.pdf_array_get(o, i+1))
+                point = mupdf.FzPoint(x, y)
+                point = mupdf.fz_transform_point(point, page_ctm)
+                res.append( (point.x, point.y))
+            return res
+            
+        o = mupdf.pdf_dict_gets(annot_obj, 'InkList')
+        if o.m_internal:
+            # InkList has 2-level lists
+            #inklist:
+            res = []
+            for i in range(mupdf.pdf_array_len(o)):
+                res1 = []
+                o1 = mupdf.pdf_array_get(o, i)
+                for j in range(0, mupdf.pdf_array_len(o1), 2):
+                    x = mupdf.pdf_to_real(mupdf.pdf_array_get(o1, j))
+                    y = mupdf.pdf_to_real(mupdf.pdf_array_get(o1, j+1))
+                    point = mupdf.FzPoint(x, y)
+                    point = mupdf.fz_transform_point(point, page_ctm)
+                    res1.append( (point.x, point.y))
+                res.append(res1)
+            return res
+
+    @property
+    def xref(self):
+        """annotation xref number"""
+        CheckParent(self)
+        annot = self.this
+        return mupdf.pdf_to_num(mupdf.pdf_annot_obj(annot))
+
+
+class Archive:
+    def __init__( self, *args):
+        '''
+        Archive(dirname [, path]) - from folder
+        Archive(file [, path]) - from file name or object
+        Archive(data, name) - from memory item
+        Archive() - empty archive
+        Archive(archive [, path]) - from archive
+        '''
+        self._subarchives = list()
+        self.this = mupdf.fz_new_multi_archive()
+        if args:
+            self.add( *args)
+    
+    def __repr__( self):
+        return f'Archive, sub-archives: {len(self._subarchives)}'
+
+    def _add_arch( self, subarch, path=None):
+        mupdf.fz_mount_multi_archive( self.this, subarch, path)
+    
+    def _add_dir( self, folder, path=None):
+        sub = mupdf.fz_open_directory( folder)
+        mupdf.fz_mount_multi_archive( self.this, sub, path)
+    
+    def _add_treeitem( self, memory, name, path=None):
+        buff = JM_BufferFromBytes( memory)
+        sub = mupdf.fz_new_tree_archive( mupdf.FzTree())
+        mupdf.fz_tree_archive_add_buffer( sub, name, buff)
+        mupdf.fz_mount_multi_archive( self.this, sub, path)
+    
+    def _add_ziptarfile( self, filepath, type_, path=None):
+        if type_ == 1:
+            sub = mupdf.fz_open_zip_archive( filepath)
+        else:
+            sub = mupdf.fz_open_tar_archive( filepath)
+        mupdf.fz_mount_multi_archive( self.this, sub, path)
+    
+    def _add_ziptarmemory( self, memory, type_, path=None):
+        buff = JM_BufferFromBytes( memory)
+        stream = mupdf.fz_open_buffer( buff)
+        if type_==1:
+            sub = mupdf.fz_open_zip_archive_with_stream( stream)
+        else:
+            sub = mupdf.fz_open_tar_archive_with_stream( stream)
+        mupdf.fz_mount_multi_archive( self.this, sub, path)
+    
+    def add( self, content, path=None):
+        '''
+        Add a sub-archive.
+
+        Args:
+            content:
+                The content to be added. May be one of:
+                    `str` - must be path of directory or file.
+                    `bytes`, `bytearray`, `io.BytesIO` - raw data.
+                    `zipfile.Zipfile`.
+                    `tarfile.TarFile`.
+                    `pymupdf.Archive`.
+                    A two-item tuple `(data, name)`.
+                    List or tuple (but not tuple with length 2) of the above.
+            path: (str) a "virtual" path name, under which the elements
+                of content can be retrieved. Use it to e.g. cope with
+                duplicate element names.
+        '''
+        def is_binary_data(x):
+            return isinstance(x, (bytes, bytearray, io.BytesIO))
+
+        def make_subarch(entries, mount, fmt):
+            subarch = dict(fmt=fmt, entries=entries, path=mount)
+            if fmt != "tree" or self._subarchives == []:
+                self._subarchives.append(subarch)
+            else:
+                ltree = self._subarchives[-1]
+                if ltree["fmt"] != "tree" or ltree["path"] != subarch["path"]:
+                    self._subarchives.append(subarch)
+                else:
+                    ltree["entries"].extend(subarch["entries"])
+                    self._subarchives[-1] = ltree
+
+        if isinstance(content, pathlib.Path):
+            content = str(content)
+        
+        if isinstance(content, str):
+            if os.path.isdir(content):
+                self._add_dir(content, path)
+                return make_subarch(os.listdir(content), path, 'dir')
+            elif os.path.isfile(content):
+                assert isinstance(path, str) and path != '', \
+                        f'Need name for binary content, but {path=}.'
+                with open(content) as f:
+                    ff = f.read()
+                self._add_treeitem(ff, path)
+                return make_subarch([path], None, 'tree')
+            else:
+                raise ValueError(f'Not a file or directory: {content!r}')
+
+        elif is_binary_data(content):
+            assert isinstance(path, str) and path != '' \
+                    f'Need name for binary content, but {path=}.'
+            self._add_treeitem(content, path)
+            return make_subarch([path], None, 'tree')
+
+        elif isinstance(content, zipfile.ZipFile):
+            filename = getattr(content, "filename", None)
+            if filename is None:
+                fp = content.fp.getvalue()
+                self._add_ziptarmemory(fp, 1, path)
+            else:
+                self._add_ziptarfile(filename, 1, path)
+            return make_subarch(content.namelist(), path, 'zip')
+
+        elif isinstance(content, tarfile.TarFile):
+            filename = getattr(content.fileobj, "name", None)
+            if filename is None:
+                fp = content.fileobj
+                if not isinstance(fp, io.BytesIO):
+                    fp = fp.fileobj
+                self._add_ziptarmemory(fp.getvalue(), 0, path)
+            else:
+                self._add_ziptarfile(filename, 0, path)
+            return make_subarch(content.getnames(), path, 'tar')
+
+        elif isinstance(content, Archive):
+            self._add_arch(content, path)
+            return make_subarch([], path, 'multi')
+        
+        if isinstance(content, tuple) and len(content) == 2:
+            # covers the tree item plus path
+            data, name = content
+            assert isinstance(name, str), f'Unexpected {type(name)=}'
+            if is_binary_data(data):
+                self._add_treeitem(data, name, path=path)
+            elif isinstance(data, str):
+                if os.path.isfile(data):
+                    with open(data, 'rb') as f:
+                        ff = f.read()
+                    self._add_treeitem(ff, name, path=path)
+            else:
+                assert 0, f'Unexpected {type(data)=}.'
+            return make_subarch([name], path, 'tree')
+        
+        elif hasattr(content, '__getitem__'):
+            # Deal with sequence of disparate items.
+            for item in content:
+                self.add(item, path)
+            return
+        
+        else:
+            raise TypeError(f'Unrecognised type {type(content)}.')
+        assert 0
+
+    @property
+    def entry_list( self):
+        '''
+        List of sub archives.
+        '''
+        return self._subarchives
+    
+    def has_entry( self, name):
+        return mupdf.fz_has_archive_entry( self.this, name)
+    
+    def read_entry( self, name):
+        buff = mupdf.fz_read_archive_entry( self.this, name)
+        return JM_BinFromBuffer( buff)
+
+
+class Xml:
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        pass
+
+    def __init__(self, rhs):
+        if isinstance(rhs, mupdf.FzXml):
+            self.this = rhs
+        elif isinstance(rhs, str):
+            buff = mupdf.fz_new_buffer_from_copied_data(rhs)
+            self.this = mupdf.fz_parse_xml_from_html5(buff)
+        else:
+            assert 0, f'Unsupported type for rhs: {type(rhs)}'
+    
+    def _get_node_tree( self):
+        def show_node(node, items, shift):
+            while node is not None:
+                if node.is_text:
+                    items.append((shift, f'"{node.text}"'))
+                    node = node.next
+                    continue
+                items.append((shift, f"({node.tagname}"))
+                for k, v in node.get_attributes().items():
+                    items.append((shift, f"={k} '{v}'"))
+                child = node.first_child
+                if child:
+                    items = show_node(child, items, shift + 1)
+                items.append((shift, f"){node.tagname}"))
+                node = node.next
+            return items
+
+        shift = 0
+        items = []
+        items = show_node(self, items, shift)
+        return items
+    
+    def add_bullet_list(self):
+        """Add bulleted list ("ul" tag)"""
+        child = self.create_element("ul")
+        self.append_child(child)
+        return child
+
+    def add_class(self, text):
+        """Set some class via CSS. Replaces complete class spec."""
+        cls = self.get_attribute_value("class")
+        if cls is not None and text in cls:
+            return self
+        self.remove_attribute("class")
+        if cls is None:
+            cls = text
+        else:
+            cls += " " + text
+        self.set_attribute("class", cls)
+        return self
+
+    def add_code(self, text=None):
+        """Add a "code" tag"""
+        child = self.create_element("code")
+        if type(text) is str:
+            child.append_child(self.create_text_node(text))
+        prev = self.span_bottom()
+        if prev is None:
+            prev = self
+        prev.append_child(child)
+        return self
+
+    def add_codeblock(self):
+        """Add monospaced lines ("pre" node)"""
+        child = self.create_element("pre")
+        self.append_child(child)
+        return child
+
+    def add_description_list(self):
+        """Add description list ("dl" tag)"""
+        child = self.create_element("dl")
+        self.append_child(child)
+        return child
+
+    def add_division(self):
+        """Add "div" tag"""
+        child = self.create_element("div")
+        self.append_child(child)
+        return child
+
+    def add_header(self, level=1):
+        """Add header tag"""
+        if level not in range(1, 7):
+            raise ValueError("Header level must be in [1, 6]")
+        this_tag = self.tagname
+        new_tag = f"h{level}"
+        child = self.create_element(new_tag)
+        if this_tag not in ("h1", "h2", "h3", "h4", "h5", "h6", "p"):
+            self.append_child(child)
+            return child
+        self.parent.append_child(child)
+        return child
+
+    def add_horizontal_line(self):
+        """Add horizontal line ("hr" tag)"""
+        child = self.create_element("hr")
+        self.append_child(child)
+        return child
+
+    def add_image(self, name, width=None, height=None, imgfloat=None, align=None):
+        """Add image node (tag "img")."""
+        child = self.create_element("img")
+        if width is not None:
+            child.set_attribute("width", f"{width}")
+        if height is not None:
+            child.set_attribute("height", f"{height}")
+        if imgfloat is not None:
+            child.set_attribute("style", f"float: {imgfloat}")
+        if align is not None:
+            child.set_attribute("align", f"{align}")
+        child.set_attribute("src", f"{name}")
+        self.append_child(child)
+        return child
+
+    def add_link(self, href, text=None):
+        """Add a hyperlink ("a" tag)"""
+        child = self.create_element("a")
+        if not isinstance(text, str):
+            text = href
+        child.set_attribute("href", href)
+        child.append_child(self.create_text_node(text))
+        prev = self.span_bottom()
+        if prev is None:
+            prev = self
+        prev.append_child(child)
+        return self
+
+    def add_list_item(self):
+        """Add item ("li" tag) under a (numbered or bulleted) list."""
+        if self.tagname not in ("ol", "ul"):
+            raise ValueError("cannot add list item to", self.tagname)
+        child = self.create_element("li")
+        self.append_child(child)
+        return child
+
+    def add_number_list(self, start=1, numtype=None):
+        """Add numbered list ("ol" tag)"""
+        child = self.create_element("ol")
+        if start > 1:
+            child.set_attribute("start", str(start))
+        if numtype is not None:
+            child.set_attribute("type", numtype)
+        self.append_child(child)
+        return child
+
+    def add_paragraph(self):
+        """Add "p" tag"""
+        child = self.create_element("p")
+        if self.tagname != "p":
+            self.append_child(child)
+        else:
+            self.parent.append_child(child)
+        return child
+
+    def add_span(self):
+        child = self.create_element("span")
+        self.append_child(child)
+        return child
+
+    def add_style(self, text):
+        """Set some style via CSS style. Replaces complete style spec."""
+        style = self.get_attribute_value("style")
+        if style is not None and text in style:
+            return self
+        self.remove_attribute("style")
+        if style is None:
+            style = text
+        else:
+            style += ";" + text
+        self.set_attribute("style", style)
+        return self
+
+    def add_subscript(self, text=None):
+        """Add a subscript ("sub" tag)"""
+        child = self.create_element("sub")
+        if type(text) is str:
+            child.append_child(self.create_text_node(text))
+        prev = self.span_bottom()
+        if prev is None:
+            prev = self
+        prev.append_child(child)
+        return self
+
+    def add_superscript(self, text=None):
+        """Add a superscript ("sup" tag)"""
+        child = self.create_element("sup")
+        if type(text) is str:
+            child.append_child(self.create_text_node(text))
+        prev = self.span_bottom()
+        if prev is None:
+            prev = self
+        prev.append_child(child)
+        return self
+
+    def add_text(self, text):
+        """Add text. Line breaks are honored."""
+        lines = text.splitlines()
+        line_count = len(lines)
+        prev = self.span_bottom()
+        if prev is None:
+            prev = self
+
+        for i, line in enumerate(lines):
+            prev.append_child(self.create_text_node(line))
+            if i < line_count - 1:
+                prev.append_child(self.create_element("br"))
+        return self
+
+    def append_child( self, child):
+        mupdf.fz_dom_append_child( self.this, child.this)
+    
+    def append_styled_span(self, style):
+        span = self.create_element("span")
+        span.add_style(style)
+        prev = self.span_bottom()
+        if prev is None:
+            prev = self
+        prev.append_child(span)
+        return prev
+
+    def bodytag( self):
+        return Xml( mupdf.fz_dom_body( self.this))
+    
+    def clone( self):
+        ret = mupdf.fz_dom_clone( self.this)
+        return Xml( ret)
+    
+    @staticmethod
+    def color_text(color):
+        if type(color) is str:
+            return color
+        if type(color) is int:
+            return f"rgb({sRGB_to_rgb(color)})"
+        if type(color) in (tuple, list):
+            return f"rgb{tuple(color)}"
+        return color
+
+    def create_element( self, tag):
+        return Xml( mupdf.fz_dom_create_element( self.this, tag))
+    
+    def create_text_node( self, text):
+        return Xml( mupdf.fz_dom_create_text_node( self.this, text))
+    
+    def debug(self):
+        """Print a list of the node tree below self."""
+        items = self._get_node_tree()
+        for item in items:
+            message("  " * item[0] + item[1].replace("\n", "\\n"))
+
+    def find( self, tag, att, match):
+        ret = mupdf.fz_dom_find( self.this, tag, att, match)
+        if ret.m_internal:
+            return Xml( ret)
+    
+    def find_next( self, tag, att, match):
+        ret = mupdf.fz_dom_find_next( self.this, tag, att, match)
+        if ret.m_internal:
+            return Xml( ret)
+    
+    @property
+    def first_child( self):
+        if mupdf.fz_xml_text( self.this):
+            # text node, has no child.
+            return
+        ret = mupdf.fz_dom_first_child( self)
+        if ret.m_internal:
+            return Xml( ret)
+    
+    def get_attribute_value( self, key):
+        assert key
+        return mupdf.fz_dom_attribute( self.this, key)
+    
+    def get_attributes( self):
+        if mupdf.fz_xml_text( self.this):
+            # text node, has no attributes.
+            return
+        result = dict()
+        i = 0
+        while 1:
+            val, key = mupdf.fz_dom_get_attribute( self.this, i)
+            if not val or not key:
+                break
+            result[ key] = val
+            i += 1
+        return result
+    
+    def insert_after( self, node):
+        mupdf.fz_dom_insert_after( self.this, node.this)
+    
+    def insert_before( self, node):
+        mupdf.fz_dom_insert_before( self.this, node.this)
+    
+    def insert_text(self, text):
+        lines = text.splitlines()
+        line_count = len(lines)
+        for i, line in enumerate(lines):
+            self.append_child(self.create_text_node(line))
+            if i < line_count - 1:
+                self.append_child(self.create_element("br"))
+        return self
+
+    @property
+    def is_text(self):
+        """Check if this is a text node."""
+        return self.text is not None
+
+    @property
+    def last_child(self):
+        """Return last child node."""
+        child = self.first_child
+        if child is None:
+            return None
+        while True:
+            next = child.next
+            if not next:
+                return child
+            child = next
+
+    @property
+    def next( self):
+        ret = mupdf.fz_dom_next( self.this)
+        if ret.m_internal:
+            return Xml( ret)
+            
+    @property
+    def parent( self):
+        ret = mupdf.fz_dom_parent( self.this)
+        if ret.m_internal:
+            return Xml( ret)
+    
+    @property
+    def previous( self):
+        ret = mupdf.fz_dom_previous( self.this)
+        if ret.m_internal:
+            return Xml( ret)
+    
+    def remove( self):
+        mupdf.fz_dom_remove( self.this)
+    
+    def remove_attribute( self, key):
+        assert key
+        mupdf.fz_dom_remove_attribute( self.this, key)
+    
+    @property
+    def root( self):
+        return Xml( mupdf.fz_xml_root( self.this))
+    
+    def set_align(self, align):
+        """Set text alignment via CSS style"""
+        text = "text-align: %s"
+        if isinstance( align, str):
+            t = align
+        elif align == TEXT_ALIGN_LEFT:
+            t = "left"
+        elif align == TEXT_ALIGN_CENTER:
+            t = "center"
+        elif align == TEXT_ALIGN_RIGHT:
+            t = "right"
+        elif align == TEXT_ALIGN_JUSTIFY:
+            t = "justify"
+        else:
+            raise ValueError(f"Unrecognised {align=}")
+        text = text % t
+        self.add_style(text)
+        return self
+
+    def set_attribute( self, key, value):
+        assert key
+        mupdf.fz_dom_add_attribute( self.this, key, value)
+    
+    def set_bgcolor(self, color):
+        """Set background color via CSS style"""
+        text = f"background-color: %s" % self.color_text(color)
+        self.add_style(text)  # does not work on span level
+        return self
+
+    def set_bold(self, val=True):
+        """Set bold on / off via CSS style"""
+        if val:
+            val="bold"
+        else:
+            val="normal"
+        text = "font-weight: %s" % val
+        self.append_styled_span(text)
+        return self
+
+    def set_color(self, color):
+        """Set text color via CSS style"""
+        text = f"color: %s" % self.color_text(color)
+        self.append_styled_span(text)
+        return self
+
+    def set_columns(self, cols):
+        """Set number of text columns via CSS style"""
+        text = f"columns: {cols}"
+        self.append_styled_span(text)
+        return self
+
+    def set_font(self, font):
+        """Set font-family name via CSS style"""
+        text = "font-family: %s" % font
+        self.append_styled_span(text)
+        return self
+
+    def set_fontsize(self, fontsize):
+        """Set font size name via CSS style"""
+        if type(fontsize) is str:
+            px=""
+        else:
+            px="px"
+        text = f"font-size: {fontsize}{px}"
+        self.append_styled_span(text)
+        return self
+
+    def set_id(self, unique):
+        """Set a unique id."""
+        # check uniqueness
+        root = self.root
+        if root.find(None, "id", unique):
+            raise ValueError(f"id '{unique}' already exists")
+        self.set_attribute("id", unique)
+        return self
+
+    def set_italic(self, val=True):
+        """Set italic on / off via CSS style"""
+        if val:
+            val="italic"
+        else:
+            val="normal"
+        text = "font-style: %s" % val
+        self.append_styled_span(text)
+        return self
+
+    def set_leading(self, leading):
+        """Set inter-line spacing value via CSS style - block-level only."""
+        text = f"-mupdf-leading: {leading}"
+        self.add_style(text)
+        return self
+
+    def set_letter_spacing(self, spacing):
+        """Set inter-letter spacing value via CSS style"""
+        text = f"letter-spacing: {spacing}"
+        self.append_styled_span(text)
+        return self
+
+    def set_lineheight(self, lineheight):
+        """Set line height name via CSS style - block-level only."""
+        text = f"line-height: {lineheight}"
+        self.add_style(text)
+        return self
+
+    def set_margins(self, val):
+        """Set margin values via CSS style"""
+        text = "margins: %s" % val
+        self.append_styled_span(text)
+        return self
+
+    def set_opacity(self, opacity):
+        """Set opacity via CSS style"""
+        text = f"opacity: {opacity}"
+        self.append_styled_span(text)
+        return self
+
+    def set_pagebreak_after(self):
+        """Insert a page break after this node."""
+        text = "page-break-after: always"
+        self.add_style(text)
+        return self
+
+    def set_pagebreak_before(self):
+        """Insert a page break before this node."""
+        text = "page-break-before: always"
+        self.add_style(text)
+        return self
+
+    def set_properties(
+            self,
+            align=None,
+            bgcolor=None,
+            bold=None,
+            color=None,
+            columns=None,
+            font=None,
+            fontsize=None,
+            indent=None,
+            italic=None,
+            leading=None,
+            letter_spacing=None,
+            lineheight=None,
+            margins=None,
+            pagebreak_after=None,
+            pagebreak_before=None,
+            word_spacing=None,
+            unqid=None,
+            cls=None,
+            ):
+        """Set any or all properties of a node.
+
+        To be used for existing nodes preferably.
+        """
+        root = self.root
+        temp = root.add_division()
+        if align is not None:
+            temp.set_align(align)
+        if bgcolor is not None:
+            temp.set_bgcolor(bgcolor)
+        if bold is not None:
+            temp.set_bold(bold)
+        if color is not None:
+            temp.set_color(color)
+        if columns is not None:
+            temp.set_columns(columns)
+        if font is not None:
+            temp.set_font(font)
+        if fontsize is not None:
+            temp.set_fontsize(fontsize)
+        if indent is not None:
+            temp.set_text_indent(indent)
+        if italic is not None:
+            temp.set_italic(italic)
+        if leading is not None:
+            temp.set_leading(leading)
+        if letter_spacing is not None:
+            temp.set_letter_spacing(letter_spacing)
+        if lineheight is not None:
+            temp.set_lineheight(lineheight)
+        if margins is not None:
+            temp.set_margins(margins)
+        if pagebreak_after is not None:
+            temp.set_pagebreak_after()
+        if pagebreak_before is not None:
+            temp.set_pagebreak_before()
+        if word_spacing is not None:
+            temp.set_word_spacing(word_spacing)
+        if unqid is not None:
+            self.set_id(unqid)
+        if cls is not None:
+            self.add_class(cls)
+
+        styles = []
+        top_style = temp.get_attribute_value("style")
+        if top_style is not None:
+            styles.append(top_style)
+        child = temp.first_child
+        while child:
+            styles.append(child.get_attribute_value("style"))
+            child = child.first_child
+        self.set_attribute("style", ";".join(styles))
+        temp.remove()
+        return self
+
+    def set_text_indent(self, indent):
+        """Set text indentation name via CSS style - block-level only."""
+        text = f"text-indent: {indent}"
+        self.add_style(text)
+        return self
+
+    def set_underline(self, val="underline"):
+        text = "text-decoration: %s" % val
+        self.append_styled_span(text)
+        return self
+
+    def set_word_spacing(self, spacing):
+        """Set inter-word spacing value via CSS style"""
+        text = f"word-spacing: {spacing}"
+        self.append_styled_span(text)
+        return self
+
+    def span_bottom(self):
+        """Find deepest level in stacked spans."""
+        parent = self
+        child = self.last_child
+        if child is None:
+            return None
+        while child.is_text:
+            child = child.previous
+            if child is None:
+                break
+        if child is None or child.tagname != "span":
+            return None
+
+        while True:
+            if child is None:
+                return parent
+            if child.tagname in ("a", "sub","sup","body") or child.is_text:
+                child = child.next
+                continue
+            if child.tagname == "span":
+                parent = child
+                child = child.first_child
+            else:
+                return parent
+
+    @property
+    def tagname( self):
+        return mupdf.fz_xml_tag( self.this)
+    
+    @property
+    def text( self):
+        return mupdf.fz_xml_text( self.this)
+    
+    add_var = add_code
+    add_samp = add_code
+    add_kbd = add_code
+
+
+class Colorspace:
+
+    def __init__(self, type_):
+        """Supported are GRAY, RGB and CMYK."""
+        if isinstance( type_, mupdf.FzColorspace):
+            self.this = type_
+        elif type_ == CS_GRAY:
+            self.this = mupdf.FzColorspace(mupdf.FzColorspace.Fixed_GRAY)
+        elif type_ == CS_CMYK:
+            self.this = mupdf.FzColorspace(mupdf.FzColorspace.Fixed_CMYK)
+        elif type_ == CS_RGB:
+            self.this = mupdf.FzColorspace(mupdf.FzColorspace.Fixed_RGB)
+        else:
+            self.this = mupdf.FzColorspace(mupdf.FzColorspace.Fixed_RGB)
+
+    def __repr__(self):
+        x = ("", "GRAY", "", "RGB", "CMYK")[self.n]
+        return "Colorspace(CS_%s) - %s" % (x, self.name)
+
+    def _name(self):
+        return mupdf.fz_colorspace_name(self.this)
+
+    @property
+    def n(self):
+        """Size of one pixel."""
+        return mupdf.fz_colorspace_n(self.this)
+
+    @property
+    def name(self):
+        """Name of the Colorspace."""
+        return self._name()
+
+
+class DeviceWrapper:
+    def __init__(self, *args):
+        if args_match( args, mupdf.FzDevice):
+            device, = args
+            self.this = device
+        elif args_match( args, Pixmap, None):
+            pm, clip = args
+            bbox = JM_irect_from_py( clip)
+            if mupdf.fz_is_infinite_irect( bbox):
+                self.this = mupdf.fz_new_draw_device( mupdf.FzMatrix(), pm)
+            else:
+                self.this = mupdf.fz_new_draw_device_with_bbox( mupdf.FzMatrix(), pm, bbox)
+        elif args_match( args, mupdf.FzDisplayList):
+            dl, = args
+            self.this = mupdf.fz_new_list_device( dl)
+        elif args_match( args, mupdf.FzStextPage, None):
+            tp, flags = args
+            opts = mupdf.FzStextOptions( flags)
+            self.this = mupdf.fz_new_stext_device( tp, opts)
+        else:
+            raise Exception( f'Unrecognised args for DeviceWrapper: {args!r}')
+
+
+class DisplayList:
+    def __del__(self):
+        if not type(self) is DisplayList: return
+        self.thisown = False
+
+    def __init__(self, *args):
+        if len(args) == 1 and isinstance(args[0], mupdf.FzRect):
+            self.this = mupdf.FzDisplayList(args[0])
+        elif len(args) == 1 and isinstance(args[0], mupdf.FzDisplayList):
+            self.this = args[0]
+        else:
+            assert 0, f'Unrecognised {args=}'
+
+    def get_pixmap(self, matrix=None, colorspace=None, alpha=0, clip=None):
+        if isinstance(colorspace, Colorspace):
+            colorspace = colorspace.this
+        else:
+            colorspace = mupdf.FzColorspace(mupdf.FzColorspace.Fixed_RGB)
+        val = JM_pixmap_from_display_list(self.this, matrix, colorspace, alpha, clip, None)
+        val.thisown = True
+        return val
+
+    def get_textpage(self, flags=3):
+        """Make a TextPage from a DisplayList."""
+        stext_options = mupdf.FzStextOptions()
+        stext_options.flags = flags
+        val = mupdf.FzStextPage(self.this, stext_options)
+        val.thisown = True
+        return val
+
+    @property
+    def rect(self):
+        val = JM_py_from_rect(mupdf.fz_bound_display_list(self.this))
+        val = Rect(val)
+        return val
+
+    def run(self, dw, m, area):
+        mupdf.fz_run_display_list(
+                self.this,
+                dw.device,
+                JM_matrix_from_py(m),
+                JM_rect_from_py(area),
+                mupdf.FzCookie(),
+                )
+
+if g_use_extra:
+    extra_FzDocument_insert_pdf = extra.FzDocument_insert_pdf
+
+
+class Document:
+
+    def __contains__(self, loc) -> bool:
+        if type(loc) is int:
+            if loc < self.page_count:
+                return True
+            return False
+        if type(loc) not in (tuple, list) or len(loc) != 2:
+            return False
+        chapter, pno = loc
+        if (0
+                or not isinstance(chapter, int)
+                or chapter < 0
+                or chapter >= self.chapter_count
+                ):
+            return False
+        if (0
+                or not isinstance(pno, int)
+                or pno < 0
+                or pno >= self.chapter_page_count(chapter)
+                ):
+            return False
+        return True
+
+    def __delitem__(self, i)->None:
+        if not self.is_pdf:
+            raise ValueError("is no PDF")
+        if type(i) is int:
+            return self.delete_page(i)
+        if type(i) in (list, tuple, range):
+            return self.delete_pages(i)
+        if type(i) is not slice:
+            raise ValueError("bad argument type")
+        pc = self.page_count
+        start = i.start if i.start else 0
+        stop = i.stop if i.stop else pc
+        step = i.step if i.step else 1
+        while start < 0:
+            start += pc
+        if start >= pc:
+            raise ValueError("bad page number(s)")
+        while stop < 0:
+            stop += pc
+        if stop > pc:
+            raise ValueError("bad page number(s)")
+        return self.delete_pages(range(start, stop, step))
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+    @typing.overload
+    def __getitem__(self, i: int = 0) -> Page:
+        ...
+    
+    if sys.version_info >= (3, 9):
+        @typing.overload
+        def __getitem__(self, i: slice) -> list[Page]:
+            ...
+        
+        @typing.overload
+        def __getitem__(self, i: tuple[int, int]) -> Page:
+            ...
+    
+    def __getitem__(self, i=0):
+        if isinstance(i, slice):
+            return [self[j] for j in range(*i.indices(len(self)))]
+        assert isinstance(i, int) or (isinstance(i, tuple) and len(i) == 2 and all(isinstance(x, int) for x in i)), \
+                f'Invalid item number: {i=}.'
+        if i not in self:
+            raise IndexError(f"page {i} not in document")
+        return self.load_page(i)
+
+    def __init__(self, filename=None, stream=None, filetype=None, rect=None, width=0, height=0, fontsize=11):
+        """Creates a document. Use 'open' as a synonym.
+
+        Notes:
+            Basic usages:
+            open() - new PDF document
+            open(filename) - string or pathlib.Path, must have supported
+                    file extension.
+            open(type, buffer) - type: valid extension, buffer: bytes object.
+            open(stream=buffer, filetype=type) - keyword version of previous.
+            open(filename, fileype=type) - filename with unrecognized extension.
+            rect, width, height, fontsize: layout reflowable document
+            on open (e.g. EPUB). Ignored if n/a.
+        """
+        # We temporarily set JM_mupdf_show_errors=0 while we are constructing,
+        # then restore its original value in a `finally:` block.
+        #
+        global JM_mupdf_show_errors
+        JM_mupdf_show_errors_old = JM_mupdf_show_errors
+        JM_mupdf_show_errors = 0
+        
+        try:
+            self.is_closed    = False
+            self.is_encrypted = False
+            self.is_encrypted = False
+            self.metadata    = None
+            self.FontInfos   = []
+            self.Graftmaps   = {}
+            self.ShownPages  = {}
+            self.InsertedImages  = {}
+            self._page_refs  = weakref.WeakValueDictionary()
+            if isinstance(filename, mupdf.PdfDocument):
+                pdf_document = filename
+                self.this = pdf_document
+                self.this_is_pdf = True
+                return
+        
+            w = width
+            h = height
+            r = JM_rect_from_py(rect)
+            if not mupdf.fz_is_infinite_rect(r):
+                w = r.x1 - r.x0
+                h = r.y1 - r.y0
+
+            self._name = filename
+            self.stream = stream
+            
+            if stream is not None:
+                if filename is not None and filetype is None:
+                    # 2025-05-06: Use  as the filetype. This is
+                    # reversing precedence - we used to use  if both
+                    # were set.
+                    filetype = filename
+                if isinstance(stream, (bytes, memoryview)):
+                    pass
+                elif isinstance(stream, bytearray):
+                    stream = bytes(stream)
+                elif isinstance(stream, io.BytesIO):
+                    stream = stream.getvalue()
+                else:
+                    raise TypeError(f"bad stream: {type(stream)=}.")
+                self.stream = stream
+                
+                assert isinstance(stream, (bytes, memoryview))
+                if len(stream) == 0:
+                    # MuPDF raise an exception for this but also generates
+                    # warnings, which is not very helpful for us. So instead we
+                    # raise a specific exception.
+                    raise EmptyFileError('Cannot open empty stream.')
+                    
+                stream2 = mupdf.fz_open_memory(mupdf.python_buffer_data(stream), len(stream))
+                try:
+                    doc = mupdf.fz_open_document_with_stream(filetype if filetype else '', stream2)
+                except Exception as e:
+                    if g_exceptions_verbose > 1:    exception_info()
+                    raise FileDataError('Failed to open stream') from e
+            
+            elif filename:
+                assert not stream
+                if isinstance(filename, str):
+                    pass
+                elif hasattr(filename, "absolute"):
+                    filename = str(filename)
+                elif hasattr(filename, "name"):
+                    filename = filename.name
+                else:
+                    raise TypeError(f"bad filename: {type(filename)=} {filename=}.")
+                self._name = filename
+                
+                # Generate our own specific exceptions. This avoids MuPDF
+                # generating warnings etc.
+                if not os.path.exists(filename):
+                    raise FileNotFoundError(f"no such file: '{filename}'")
+                elif not os.path.isfile(filename):
+                    raise FileDataError(f"'{filename}' is no file")
+                elif os.path.getsize(filename) == 0:
+                    raise EmptyFileError(f'Cannot open empty file: {filename=}.')
+                
+                if filetype:
+                    # Override the type implied by . MuPDF does not
+                    # have a way to do this directly so we open via a stream.
+                    try:
+                        fz_stream = mupdf.fz_open_file(filename)
+                        doc = mupdf.fz_open_document_with_stream(filetype, fz_stream)
+                    except Exception as e:
+                        if g_exceptions_verbose > 1:    exception_info()
+                        raise FileDataError(f'Failed to open file {filename!r} as type {filetype!r}.') from e
+                else:
+                    try:
+                        doc = mupdf.fz_open_document(filename)
+                    except Exception as e:
+                        if g_exceptions_verbose > 1:    exception_info()
+                        raise FileDataError(f'Failed to open file {filename!r}.') from e
+
+            else:
+                pdf = mupdf.PdfDocument()
+                doc = mupdf.FzDocument(pdf)
+            
+            if w > 0 and h > 0:
+                mupdf.fz_layout_document(doc, w, h, fontsize)
+            elif mupdf.fz_is_document_reflowable(doc):
+                mupdf.fz_layout_document(doc, 400, 600, 11)
+
+            self.this = doc
+
+            # fixme: not sure where self.thisown gets initialised in PyMuPDF.
+            #
+            self.thisown = True
+
+            if self.thisown:
+                self._graft_id = TOOLS.gen_id()
+                if self.needs_pass:
+                    self.is_encrypted = True
+                else: # we won't init until doc is decrypted
+                    self.init_doc()
+                # the following hack detects invalid/empty SVG files, which else may lead
+                # to interpreter crashes
+                if filename and filename.lower().endswith("svg") or filetype and "svg" in filetype.lower():
+                    try:
+                        _ = self.convert_to_pdf()  # this seems to always work
+                    except Exception as e:
+                        if g_exceptions_verbose > 1:    exception_info()
+                        raise FileDataError("cannot open broken document") from e
+
+            if g_use_extra:
+                self.this_is_pdf = isinstance( self.this, mupdf.PdfDocument)
+                if self.this_is_pdf:
+                    self.page_count2 = extra.page_count_pdf
+                else:
+                    self.page_count2 = extra.page_count_fz
+        finally:
+            JM_mupdf_show_errors = JM_mupdf_show_errors_old
+    
+    def __len__(self) -> int:
+        return self.page_count
+
+    def __repr__(self) -> str:
+        m = "closed " if self.is_closed else ""
+        if self.stream is None:
+            if self.name == "":
+                return m + "Document()" % self._graft_id
+            return m + "Document('%s')" % (self.name,)
+        return m + "Document('%s', )" % (self.name, self._graft_id)
+
+    def _addFormFont(self, name, font):
+        """Add new form font."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self, required=0)
+        if not pdf.m_internal:
+            return
+        fonts = mupdf.pdf_dict_getl(
+                mupdf.pdf_trailer( pdf),
+                PDF_NAME('Root'),
+                PDF_NAME('AcroForm'),
+                PDF_NAME('DR'),
+                PDF_NAME('Font'),
+                )
+        if not fonts.m_internal or not mupdf.pdf_is_dict( fonts):
+            raise RuntimeError( "PDF has no form fonts yet")
+        k = mupdf.pdf_new_name( name)
+        v = JM_pdf_obj_from_str( pdf, font)
+        mupdf.pdf_dict_put( fonts, k, v)
+
+    def _delToC(self):
+        """Delete the TOC."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        xrefs = []  # create Python list
+        pdf = _as_pdf_document(self, required=0)
+        if not pdf.m_internal:
+            return xrefs    # not a pdf
+        # get the main root
+        root = mupdf.pdf_dict_get(mupdf.pdf_trailer(pdf), PDF_NAME('Root'))
+        # get the outline root
+        olroot = mupdf.pdf_dict_get(root, PDF_NAME('Outlines'))
+        if not olroot.m_internal:
+            return xrefs    # no outlines or some problem
+
+        first = mupdf.pdf_dict_get(olroot, PDF_NAME('First'))  # first outline
+
+        xrefs = JM_outline_xrefs(first, xrefs)
+        xref_count = len(xrefs)
+
+        olroot_xref = mupdf.pdf_to_num(olroot) # delete OL root
+        mupdf.pdf_delete_object(pdf, olroot_xref)  # delete OL root
+        mupdf.pdf_dict_del(root, PDF_NAME('Outlines')) # delete OL root
+
+        for i in range(xref_count):
+            _, xref = JM_INT_ITEM(xrefs, i)
+            mupdf.pdf_delete_object(pdf, xref) # delete outline item
+        xrefs.append(olroot_xref)
+        val = xrefs
+        self.init_doc()
+        return val
+
+    def _delete_page(self, pno):
+        pdf = _as_pdf_document(self)
+        mupdf.pdf_delete_page( pdf, pno)
+        if pdf.m_internal.rev_page_map:
+            mupdf.ll_pdf_drop_page_tree( pdf.m_internal)
+
+    def _deleteObject(self, xref):
+        """Delete object."""
+        pdf = _as_pdf_document(self)
+        if not _INRANGE(xref, 1, mupdf.pdf_xref_len(pdf)-1):
+            raise ValueError( MSG_BAD_XREF)
+        mupdf.pdf_delete_object(pdf, xref)
+
+    def _embeddedFileGet(self, idx):
+        pdf = _as_pdf_document(self)
+        names = mupdf.pdf_dict_getl(
+                mupdf.pdf_trailer(pdf),
+                PDF_NAME('Root'),
+                PDF_NAME('Names'),
+                PDF_NAME('EmbeddedFiles'),
+                PDF_NAME('Names'),
+                )
+        entry = mupdf.pdf_array_get(names, 2*idx+1)
+        filespec = mupdf.pdf_dict_getl(entry, PDF_NAME('EF'), PDF_NAME('F'))
+        buf = mupdf.pdf_load_stream(filespec)
+        cont = JM_BinFromBuffer(buf)
+        return cont
+
+    def _embeddedFileIndex(self, item: typing.Union[int, str]) -> int:
+        filenames = self.embfile_names()
+        msg = "'%s' not in EmbeddedFiles array." % str(item)
+        if item in filenames:
+            idx = filenames.index(item)
+        elif item in range(len(filenames)):
+            idx = item
+        else:
+            raise ValueError(msg)
+        return idx
+
+    def _embfile_add(self, name, buffer_, filename=None, ufilename=None, desc=None):
+        pdf = _as_pdf_document(self)
+        data = JM_BufferFromBytes(buffer_)
+        if not data.m_internal:
+            raise TypeError( MSG_BAD_BUFFER)
+
+        names = mupdf.pdf_dict_getl(
+                mupdf.pdf_trailer(pdf),
+                PDF_NAME('Root'),
+                PDF_NAME('Names'),
+                PDF_NAME('EmbeddedFiles'),
+                PDF_NAME('Names'),
+                )
+        if not mupdf.pdf_is_array(names):
+            root = mupdf.pdf_dict_get(mupdf.pdf_trailer(pdf), PDF_NAME('Root'))
+            names = mupdf.pdf_new_array(pdf, 6)    # an even number!
+            mupdf.pdf_dict_putl(
+                    root,
+                    names,
+                    PDF_NAME('Names'),
+                    PDF_NAME('EmbeddedFiles'),
+                    PDF_NAME('Names'),
+                    )
+        fileentry = JM_embed_file(pdf, data, filename, ufilename, desc, 1)
+        xref = mupdf.pdf_to_num(
+                mupdf.pdf_dict_getl(fileentry, PDF_NAME('EF'), PDF_NAME('F'))
+                )
+        mupdf.pdf_array_push(names, mupdf.pdf_new_text_string(name))
+        mupdf.pdf_array_push(names, fileentry)
+        return xref
+
+    def _embfile_del(self, idx):
+        pdf = _as_pdf_document(self)
+        names = mupdf.pdf_dict_getl(
+                mupdf.pdf_trailer(pdf),
+                PDF_NAME('Root'),
+                PDF_NAME('Names'),
+                PDF_NAME('EmbeddedFiles'),
+                PDF_NAME('Names'),
+                )
+        mupdf.pdf_array_delete(names, idx + 1)
+        mupdf.pdf_array_delete(names, idx)
+
+    def _embfile_info(self, idx, infodict):
+        pdf = _as_pdf_document(self)
+        xref = 0
+        ci_xref=0
+
+        trailer = mupdf.pdf_trailer(pdf)
+
+        names = mupdf.pdf_dict_getl(
+                trailer,
+                PDF_NAME('Root'),
+                PDF_NAME('Names'),
+                PDF_NAME('EmbeddedFiles'),
+                PDF_NAME('Names'),
+                )
+        o = mupdf.pdf_array_get(names, 2*idx+1)
+        ci = mupdf.pdf_dict_get(o, PDF_NAME('CI'))
+        if ci.m_internal:
+            ci_xref = mupdf.pdf_to_num(ci)
+        infodict["collection"] = ci_xref
+        name = mupdf.pdf_to_text_string(mupdf.pdf_dict_get(o, PDF_NAME('F')))
+        infodict[dictkey_filename] = JM_EscapeStrFromStr(name)
+
+        name = mupdf.pdf_to_text_string(mupdf.pdf_dict_get(o, PDF_NAME('UF')))
+        infodict[dictkey_ufilename] = JM_EscapeStrFromStr(name)
+
+        name = mupdf.pdf_to_text_string(mupdf.pdf_dict_get(o, PDF_NAME('Desc')))
+        infodict[dictkey_descr] = JM_UnicodeFromStr(name)
+
+        len_ = -1
+        DL = -1
+        fileentry = mupdf.pdf_dict_getl(o, PDF_NAME('EF'), PDF_NAME('F'))
+        xref = mupdf.pdf_to_num(fileentry)
+        o = mupdf.pdf_dict_get(fileentry, PDF_NAME('Length'))
+        if o.m_internal:
+            len_ = mupdf.pdf_to_int(o)
+
+        o = mupdf.pdf_dict_get(fileentry, PDF_NAME('DL'))
+        if o.m_internal:
+            DL = mupdf.pdf_to_int(o)
+        else:
+            o = mupdf.pdf_dict_getl(fileentry, PDF_NAME('Params'), PDF_NAME('Size'))
+            if o.m_internal:
+                DL = mupdf.pdf_to_int(o)
+        infodict[dictkey_size] = DL
+        infodict[dictkey_length] = len_
+        return xref
+
+    def _embfile_names(self, namelist):
+        """Get list of embedded file names."""
+        pdf = _as_pdf_document(self)
+        names = mupdf.pdf_dict_getl(
+                mupdf.pdf_trailer(pdf),
+                PDF_NAME('Root'),
+                PDF_NAME('Names'),
+                PDF_NAME('EmbeddedFiles'),
+                PDF_NAME('Names'),
+                )
+        if mupdf.pdf_is_array(names):
+            n = mupdf.pdf_array_len(names)
+            for i in range(0, n, 2):
+                val = JM_EscapeStrFromStr(
+                        mupdf.pdf_to_text_string(
+                            mupdf.pdf_array_get(names, i)
+                            )
+                        )
+                namelist.append(val)
+
+    def _embfile_upd(self, idx, buffer_=None, filename=None, ufilename=None, desc=None):
+        pdf = _as_pdf_document(self)
+        xref = 0
+        names = mupdf.pdf_dict_getl(
+                mupdf.pdf_trailer(pdf),
+                PDF_NAME('Root'),
+                PDF_NAME('Names'),
+                PDF_NAME('EmbeddedFiles'),
+                PDF_NAME('Names'),
+                )
+        entry = mupdf.pdf_array_get(names, 2*idx+1)
+
+        filespec = mupdf.pdf_dict_getl(entry, PDF_NAME('EF'), PDF_NAME('F'))
+        if not filespec.m_internal:
+            RAISEPY( "bad PDF: no /EF object", JM_Exc_FileDataError)
+        res = JM_BufferFromBytes(buffer_)
+        if buffer_ and buffer_.m_internal and not res.m_internal:
+            raise TypeError( MSG_BAD_BUFFER)
+        if res.m_internal and buffer_ and buffer_.m_internal:
+            JM_update_stream(pdf, filespec, res, 1)
+            # adjust /DL and /Size parameters
+            len, _ = mupdf.fz_buffer_storage(res)
+            l = mupdf.pdf_new_int(len)
+            mupdf.pdf_dict_put(filespec, PDF_NAME('DL'), l)
+            mupdf.pdf_dict_putl(filespec, l, PDF_NAME('Params'), PDF_NAME('Size'))
+        xref = mupdf.pdf_to_num(filespec)
+        if filename:
+            mupdf.pdf_dict_put_text_string(entry, PDF_NAME('F'), filename)
+
+        if ufilename:
+            mupdf.pdf_dict_put_text_string(entry, PDF_NAME('UF'), ufilename)
+
+        if desc:
+            mupdf.pdf_dict_put_text_string(entry, PDF_NAME('Desc'), desc)
+        return xref
+
+    def _extend_toc_items(self, items):
+        """Add color info to all items of an extended TOC list."""
+        if self.is_closed:
+            raise ValueError("document closed")
+        if g_use_extra:
+            return extra.Document_extend_toc_items( self.this, items)
+        pdf = _as_pdf_document(self)
+        zoom = "zoom"
+        bold = "bold"
+        italic = "italic"
+        collapse = "collapse"
+
+        root = mupdf.pdf_dict_get(mupdf.pdf_trailer(pdf), PDF_NAME('Root'))
+        if not root.m_internal:
+            return
+        olroot = mupdf.pdf_dict_get(root, PDF_NAME('Outlines'))
+        if not olroot.m_internal:
+            return
+        first = mupdf.pdf_dict_get(olroot, PDF_NAME('First'))
+        if not first.m_internal:
+            return
+        xrefs = []
+        xrefs = JM_outline_xrefs(first, xrefs)
+        n = len(xrefs)
+        m = len(items)
+        if not n:
+            return
+        if n != m:
+            raise IndexError( "internal error finding outline xrefs")
+
+        # update all TOC item dictionaries
+        for i in range(n):
+            xref = int(xrefs[i])
+            item = items[i]
+            itemdict = item[3]
+            if not isinstance(itemdict, dict):
+                raise ValueError( "need non-simple TOC format")
+            itemdict[dictkey_xref] = xrefs[i]
+            bm = mupdf.pdf_load_object(pdf, xref)
+            flags = mupdf.pdf_to_int( mupdf.pdf_dict_get(bm, PDF_NAME('F')))
+            if flags == 1:
+                itemdict[italic] = True
+            elif flags == 2:
+                itemdict[bold] = True
+            elif flags == 3:
+                itemdict[italic] = True
+                itemdict[bold] = True
+            count = mupdf.pdf_to_int( mupdf.pdf_dict_get(bm, PDF_NAME('Count')))
+            if count < 0:
+                itemdict[collapse] = True
+            elif count > 0:
+                itemdict[collapse] = False
+            col = mupdf.pdf_dict_get(bm, PDF_NAME('C'))
+            if mupdf.pdf_is_array(col) and mupdf.pdf_array_len(col) == 3:
+                color = (
+                        mupdf.pdf_to_real(mupdf.pdf_array_get(col, 0)),
+                        mupdf.pdf_to_real(mupdf.pdf_array_get(col, 1)),
+                        mupdf.pdf_to_real(mupdf.pdf_array_get(col, 2)),
+                        )
+                itemdict[dictkey_color] = color
+            z=0
+            obj = mupdf.pdf_dict_get(bm, PDF_NAME('Dest'))
+            if not obj.m_internal or not mupdf.pdf_is_array(obj):
+                obj = mupdf.pdf_dict_getl(bm, PDF_NAME('A'), PDF_NAME('D'))
+            if mupdf.pdf_is_array(obj) and mupdf.pdf_array_len(obj) == 5:
+                z = mupdf.pdf_to_real(mupdf.pdf_array_get(obj, 4))
+            itemdict[zoom] = float(z)
+            item[3] = itemdict
+            items[i] = item
+
+    def _forget_page(self, page: Page):
+        """Remove a page from document page dict."""
+        pid = id(page)
+        if pid in self._page_refs:
+            #self._page_refs[pid] = None
+            del self._page_refs[pid]
+
+    def _get_char_widths(self, xref: int, bfname: str, ext: str, ordering: int, limit: int, idx: int = 0):
+        pdf = _as_pdf_document(self)
+        mylimit = limit
+        if mylimit < 256:
+            mylimit = 256
+        if ordering >= 0:
+            data, size, index = mupdf.fz_lookup_cjk_font(ordering)
+            font = mupdf.fz_new_font_from_memory(None, data, size, index, 0)
+        else:
+            data, size = mupdf.fz_lookup_base14_font(bfname)
+            if data:
+                font = mupdf.fz_new_font_from_memory(bfname, data, size, 0, 0)
+            else:
+                buf = JM_get_fontbuffer(pdf, xref)
+                if not buf.m_internal:
+                    raise Exception("font at xref %d is not supported" % xref)
+
+                font = mupdf.fz_new_font_from_buffer(None, buf, idx, 0)
+        wlist = []
+        for i in range(mylimit):
+            glyph = mupdf.fz_encode_character(font, i)
+            adv = mupdf.fz_advance_glyph(font, glyph, 0)
+            if ordering >= 0:
+                glyph = i
+            if glyph > 0:
+                wlist.append( (glyph, adv))
+            else:
+                wlist.append( (glyph, 0.0))
+        return wlist
+
+    def _get_page_labels(self):
+        pdf = _as_pdf_document(self)
+        rc = []
+        pagelabels = mupdf.pdf_new_name("PageLabels")
+        obj = mupdf.pdf_dict_getl( mupdf.pdf_trailer(pdf), PDF_NAME('Root'), pagelabels)
+        if not obj.m_internal:
+            return rc
+        # simple case: direct /Nums object
+        nums = mupdf.pdf_resolve_indirect( mupdf.pdf_dict_get( obj, PDF_NAME('Nums')))
+        if nums.m_internal:
+            JM_get_page_labels(rc, nums)
+            return rc
+        # case: /Kids/Nums
+        nums = mupdf.pdf_resolve_indirect( mupdf.pdf_dict_getl(obj, PDF_NAME('Kids'), PDF_NAME('Nums')))
+        if nums.m_internal:
+            JM_get_page_labels(rc, nums)
+            return rc
+        # case: /Kids is an array of multiple /Nums
+        kids = mupdf.pdf_resolve_indirect( mupdf.pdf_dict_get( obj, PDF_NAME('Kids')))
+        if not kids.m_internal or not mupdf.pdf_is_array(kids):
+            return rc
+        n = mupdf.pdf_array_len(kids)
+        for i in range(n):
+            nums = mupdf.pdf_resolve_indirect(
+                    mupdf.pdf_dict_get(
+                        mupdf.pdf_array_get(kids, i),
+                        PDF_NAME('Nums'),
+                        )
+                    )
+            JM_get_page_labels(rc, nums)
+        return rc
+
+    def _getMetadata(self, key):
+        """Get metadata."""
+        try:
+            return mupdf.fz_lookup_metadata2( self.this, key)
+        except Exception:
+            if g_exceptions_verbose > 2:    exception_info()
+            return ''
+
+    def _getOLRootNumber(self):
+        """Get xref of Outline Root, create it if missing."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        # get main root
+        root = mupdf.pdf_dict_get( mupdf.pdf_trailer( pdf), PDF_NAME('Root'))
+        # get outline root
+        olroot = mupdf.pdf_dict_get( root, PDF_NAME('Outlines'))
+        if not olroot.m_internal:
+            olroot = mupdf.pdf_new_dict( pdf, 4)
+            mupdf.pdf_dict_put( olroot, PDF_NAME('Type'), PDF_NAME('Outlines'))
+            ind_obj = mupdf.pdf_add_object( pdf, olroot)
+            mupdf.pdf_dict_put( root, PDF_NAME('Outlines'), ind_obj)
+            olroot = mupdf.pdf_dict_get( root, PDF_NAME('Outlines'))
+        return mupdf.pdf_to_num( olroot)
+
+    def _getPDFfileid(self):
+        """Get PDF file id."""
+        pdf = _as_pdf_document(self, required=0)
+        if not pdf.m_internal:
+            return
+        idlist = []
+        identity = mupdf.pdf_dict_get(mupdf.pdf_trailer(pdf), PDF_NAME('ID'))
+        if identity.m_internal:
+            n = mupdf.pdf_array_len(identity)
+            for i in range(n):
+                o = mupdf.pdf_array_get(identity, i)
+                text = mupdf.pdf_to_text_string(o)
+                hex_ = binascii.hexlify(text)
+                idlist.append(hex_)
+        return idlist
+
+    def _getPageInfo(self, pno, what):
+        """List fonts, images, XObjects used on a page."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        doc = self.this
+        pageCount = mupdf.pdf_count_pages(doc) if isinstance(doc, mupdf.PdfDocument) else mupdf.fz_count_pages(doc)
+        n = pno  # pno < 0 is allowed
+        while n < 0:
+            n += pageCount  # make it non-negative
+        if n >= pageCount:
+            raise ValueError( MSG_BAD_PAGENO)
+        pdf = _as_pdf_document(self)
+        pageref = mupdf.pdf_lookup_page_obj(pdf, n)
+        rsrc = mupdf.pdf_dict_get_inheritable(pageref, mupdf.PDF_ENUM_NAME_Resources)
+        liste = []
+        tracer = []
+        if rsrc.m_internal:
+            JM_scan_resources(pdf, rsrc, liste, what, 0, tracer)
+        return liste
+
+    def _insert_font(self, fontfile=None, fontbuffer=None):
+        '''
+        Utility: insert font from file or binary.
+        '''
+        pdf = _as_pdf_document(self)
+        if not fontfile and not fontbuffer:
+            raise ValueError( MSG_FILE_OR_BUFFER)
+        value = JM_insert_font(pdf, None, fontfile, fontbuffer, 0, 0, 0, 0, 0, -1)
+        return value
+
+    def _loadOutline(self):
+        """Load first outline."""
+        doc = self.this
+        assert isinstance( doc, mupdf.FzDocument)
+        try:
+            ol = mupdf.fz_load_outline( doc)
+        except Exception:
+            if g_exceptions_verbose > 1:    exception_info()
+            return
+        return Outline( ol)
+
+    def _make_page_map(self):
+        """Make an array page number -> page object."""
+        if self.is_closed:
+            raise ValueError("document closed")
+        assert 0, f'_make_page_map() is no-op'
+
+    def _move_copy_page(self, pno, nb, before, copy):
+        """Move or copy a PDF page reference."""
+        pdf = _as_pdf_document(self)
+        same = 0
+        # get the two page objects -----------------------------------
+        # locate the /Kids arrays and indices in each
+
+        page1, parent1, i1 = pdf_lookup_page_loc( pdf, pno)
+
+        kids1 = mupdf.pdf_dict_get( parent1, PDF_NAME('Kids'))
+
+        page2, parent2, i2 = pdf_lookup_page_loc( pdf, nb)
+        kids2 = mupdf.pdf_dict_get( parent2, PDF_NAME('Kids'))
+        if before:  # calc index of source page in target /Kids
+            pos = i2
+        else:
+            pos = i2 + 1
+
+        # same /Kids array? ------------------------------------------
+        same = mupdf.pdf_objcmp( kids1, kids2)
+
+        # put source page in target /Kids array ----------------------
+        if not copy and same != 0:  # update parent in page object
+            mupdf.pdf_dict_put( page1, PDF_NAME('Parent'), parent2)
+        mupdf.pdf_array_insert( kids2, page1, pos)
+
+        if same != 0:   # different /Kids arrays ----------------------
+            parent = parent2
+            while parent.m_internal:    # increase /Count objects in parents
+                count = mupdf.pdf_dict_get_int( parent, PDF_NAME('Count'))
+                mupdf.pdf_dict_put_int( parent, PDF_NAME('Count'), count + 1)
+                parent = mupdf.pdf_dict_get( parent, PDF_NAME('Parent'))
+            if not copy:    # delete original item
+                mupdf.pdf_array_delete( kids1, i1)
+                parent = parent1
+                while parent.m_internal:    # decrease /Count objects in parents
+                    count = mupdf.pdf_dict_get_int( parent, PDF_NAME('Count'))
+                    mupdf.pdf_dict_put_int( parent, PDF_NAME('Count'), count - 1)
+                    parent = mupdf.pdf_dict_get( parent, PDF_NAME('Parent'))
+        else:   # same /Kids array
+            if copy:    # source page is copied
+                parent = parent2
+                while parent.m_internal:    # increase /Count object in parents
+                    count = mupdf.pdf_dict_get_int( parent, PDF_NAME('Count'))
+                    mupdf.pdf_dict_put_int( parent, PDF_NAME('Count'), count + 1)
+                    parent = mupdf.pdf_dict_get( parent, PDF_NAME('Parent'))
+            else:
+                if i1 < pos:
+                    mupdf.pdf_array_delete( kids1, i1)
+                else:
+                    mupdf.pdf_array_delete( kids1, i1 + 1)
+        if pdf.m_internal.rev_page_map: # page map no longer valid: drop it
+            mupdf.ll_pdf_drop_page_tree( pdf.m_internal)
+
+        self._reset_page_refs()
+
+    def _newPage(self, pno=-1, width=595, height=842):
+        """Make a new PDF page."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        if g_use_extra:
+            extra._newPage( self.this, pno, width, height)
+        else:
+            pdf = _as_pdf_document(self)
+            mediabox = mupdf.FzRect(mupdf.FzRect.Fixed_UNIT)
+            mediabox.x1 = width
+            mediabox.y1 = height
+            contents = mupdf.FzBuffer()
+            if pno < -1:
+                raise ValueError( MSG_BAD_PAGENO)
+            # create /Resources and /Contents objects
+            #resources = pdf.add_object(pdf.new_dict(1))
+            resources = mupdf.pdf_add_new_dict(pdf, 1)
+            page_obj = mupdf.pdf_add_page( pdf, mediabox, 0, resources, contents)
+            mupdf.pdf_insert_page( pdf, pno, page_obj)
+        # fixme: pdf->dirty = 1;
+
+        self._reset_page_refs()
+        return self[pno]
+
+    def _remove_links_to(self, numbers):
+        pdf = _as_pdf_document(self)
+        _remove_dest_range(pdf, numbers)
+
+    def _remove_toc_item(self, xref):
+        # "remove" bookmark by letting it point to nowhere
+        pdf = _as_pdf_document(self)
+        item = mupdf.pdf_new_indirect(pdf, xref, 0)
+        mupdf.pdf_dict_del( item, PDF_NAME('Dest'))
+        mupdf.pdf_dict_del( item, PDF_NAME('A'))
+        color = mupdf.pdf_new_array( pdf, 3)
+        for i in range(3):
+            mupdf.pdf_array_push_real( color, 0.8)
+        mupdf.pdf_dict_put( item, PDF_NAME('C'), color)
+
+    def _reset_page_refs(self):
+        """Invalidate all pages in document dictionary."""
+        if getattr(self, "is_closed", True):
+            return
+        pages = [p for p in self._page_refs.values()]
+        for page in pages:
+            if page:
+                page._erase()
+                page = None
+        self._page_refs.clear()
+
+    def _set_page_labels(self, labels):
+        pdf = _as_pdf_document(self)
+        pagelabels = mupdf.pdf_new_name("PageLabels")
+        root = mupdf.pdf_dict_get(mupdf.pdf_trailer(pdf), PDF_NAME('Root'))
+        mupdf.pdf_dict_del(root, pagelabels)
+        mupdf.pdf_dict_putl(root, mupdf.pdf_new_array(pdf, 0), pagelabels, PDF_NAME('Nums'))
+
+        xref = self.pdf_catalog()
+        text = self.xref_object(xref, compressed=True)
+        text = text.replace("/Nums[]", "/Nums[%s]" % labels)
+        self.update_object(xref, text)
+
+    def _update_toc_item(self, xref, action=None, title=None, flags=0, collapse=None, color=None):
+        '''
+        "update" bookmark by letting it point to nowhere
+        '''
+        pdf = _as_pdf_document(self)
+        item = mupdf.pdf_new_indirect( pdf, xref, 0)
+        if title:
+            mupdf.pdf_dict_put_text_string( item, PDF_NAME('Title'), title)
+        if action:
+            mupdf.pdf_dict_del( item, PDF_NAME('Dest'))
+            obj = JM_pdf_obj_from_str( pdf, action)
+            mupdf.pdf_dict_put( item, PDF_NAME('A'), obj)
+        mupdf.pdf_dict_put_int( item, PDF_NAME('F'), flags)
+        if color:
+            c = mupdf.pdf_new_array( pdf, 3)
+            for i in range(3):
+                f = color[i]
+                mupdf.pdf_array_push_real( c, f)
+            mupdf.pdf_dict_put( item, PDF_NAME('C'), c)
+        elif color is not None:
+            mupdf.pdf_dict_del( item, PDF_NAME('C'))
+        if collapse is not None:
+            if mupdf.pdf_dict_get( item, PDF_NAME('Count')).m_internal:
+                i = mupdf.pdf_dict_get_int( item, PDF_NAME('Count'))
+                if (i < 0 and collapse is False) or (i > 0 and collapse is True):
+                    i = i * (-1)
+                    mupdf.pdf_dict_put_int( item, PDF_NAME('Count'), i)
+
+    @property
+    def FormFonts(self):
+        """Get list of field font resource names."""
+        pdf = _as_pdf_document(self, required=0)
+        if not pdf.m_internal:
+            return
+        fonts = mupdf.pdf_dict_getl(
+                mupdf.pdf_trailer(pdf),
+                PDF_NAME('Root'),
+                PDF_NAME('AcroForm'),
+                PDF_NAME('DR'),
+                PDF_NAME('Font'),
+                )
+        liste = list()
+        if fonts.m_internal and mupdf.pdf_is_dict(fonts):   # fonts exist
+            n = mupdf.pdf_dict_len(fonts)
+            for i in range(n):
+                f = mupdf.pdf_dict_get_key(fonts, i)
+                liste.append(JM_UnicodeFromStr(mupdf.pdf_to_name(f)))
+        return liste
+
+    def add_layer(self, name, creator=None, on=None):
+        """Add a new OC layer."""
+        pdf = _as_pdf_document(self)
+        JM_add_layer_config( pdf, name, creator, on)
+        mupdf.ll_pdf_read_ocg( pdf.m_internal)
+
+    def add_ocg(self, name, config=-1, on=1, intent=None, usage=None):
+        """Add new optional content group."""
+        xref = 0
+        pdf = _as_pdf_document(self)
+
+        # make the OCG
+        ocg = mupdf.pdf_add_new_dict(pdf, 3)
+        mupdf.pdf_dict_put(ocg, PDF_NAME('Type'), PDF_NAME('OCG'))
+        mupdf.pdf_dict_put_text_string(ocg, PDF_NAME('Name'), name)
+        intents = mupdf.pdf_dict_put_array(ocg, PDF_NAME('Intent'), 2)
+        if not intent:
+            mupdf.pdf_array_push(intents, PDF_NAME('View'))
+        elif not isinstance(intent, str):
+            assert 0, f'fixme: intent is not a str. {type(intent)=} {type=}'
+            #n = len(intent)
+            #for i in range(n):
+            #    item = intent[i]
+            #    c = JM_StrAsChar(item);
+            #    if (c) {
+            #        pdf_array_push(gctx, intents, pdf_new_name(gctx, c));
+            #    }
+            #    Py_DECREF(item);
+            #}
+        else:
+            mupdf.pdf_array_push(intents, mupdf.pdf_new_name(intent))
+        use_for = mupdf.pdf_dict_put_dict(ocg, PDF_NAME('Usage'), 3)
+        ci_name = mupdf.pdf_new_name("CreatorInfo")
+        cre_info = mupdf.pdf_dict_put_dict(use_for, ci_name, 2)
+        mupdf.pdf_dict_put_text_string(cre_info, PDF_NAME('Creator'), "PyMuPDF")
+        if usage:
+            mupdf.pdf_dict_put_name(cre_info, PDF_NAME('Subtype'), usage)
+        else:
+            mupdf.pdf_dict_put_name(cre_info, PDF_NAME('Subtype'), "Artwork")
+        indocg = mupdf.pdf_add_object(pdf, ocg)
+
+        # Insert OCG in the right config
+        ocp = JM_ensure_ocproperties(pdf)
+        obj = mupdf.pdf_dict_get(ocp, PDF_NAME('OCGs'))
+        mupdf.pdf_array_push(obj, indocg)
+
+        if config > -1:
+            obj = mupdf.pdf_dict_get(ocp, PDF_NAME('Configs'))
+            if not mupdf.pdf_is_array(obj):
+                raise ValueError( MSG_BAD_OC_CONFIG)
+            cfg = mupdf.pdf_array_get(obj, config)
+            if not cfg.m_internal:
+                raise ValueError( MSG_BAD_OC_CONFIG)
+        else:
+            cfg = mupdf.pdf_dict_get(ocp, PDF_NAME('D'))
+
+        obj = mupdf.pdf_dict_get(cfg, PDF_NAME('Order'))
+        if not obj.m_internal:
+            obj = mupdf.pdf_dict_put_array(cfg, PDF_NAME('Order'), 1)
+        mupdf.pdf_array_push(obj, indocg)
+        if on:
+            obj = mupdf.pdf_dict_get(cfg, PDF_NAME('ON'))
+            if not obj.m_internal:
+                obj = mupdf.pdf_dict_put_array(cfg, PDF_NAME('ON'), 1)
+        else:
+            obj =mupdf.pdf_dict_get(cfg, PDF_NAME('OFF'))
+            if not obj.m_internal:
+                obj =mupdf.pdf_dict_put_array(cfg, PDF_NAME('OFF'), 1)
+        mupdf.pdf_array_push(obj, indocg)
+
+        # let MuPDF take note: re-read OCProperties
+        mupdf.ll_pdf_read_ocg(pdf.m_internal)
+
+        xref = mupdf.pdf_to_num(indocg)
+        return xref
+
+    def authenticate(self, password):
+        """Decrypt document."""
+        if self.is_closed:
+            raise ValueError("document closed")
+        val = mupdf.fz_authenticate_password(self.this, password)
+        if val:  # the doc is decrypted successfully and we init the outline
+            self.is_encrypted = False
+            self.is_encrypted = False
+            self.init_doc()
+            self.thisown = True
+        return val
+
+    def can_save_incrementally(self):
+        """Check whether incremental saves are possible."""
+        pdf = _as_pdf_document(self, required=0)
+        if not pdf.m_internal:
+            return False
+        return mupdf.pdf_can_be_saved_incrementally(pdf)
+
+    def bake(self, *, annots: bool = True, widgets: bool = True) -> None:
+        """Convert annotations or fields to permanent content.
+
+        Notes:
+            Converts annotations or widgets to permanent page content, like
+            text and vector graphics, as appropriate.
+            After execution, pages will still look the same, but no longer
+            have annotations, respectively no fields.
+            If widgets are selected the PDF will no longer be a Form PDF.
+
+        Args:
+            annots: convert annotations
+            widgets: convert form fields
+
+        """
+        pdf = _as_pdf_document(self)
+        mupdf.pdf_bake_document(pdf, int(annots), int(widgets))
+
+    @property
+    def chapter_count(self):
+        """Number of chapters."""
+        if self.is_closed:
+            raise ValueError("document closed")
+        return mupdf.fz_count_chapters( self.this)
+
+    def chapter_page_count(self, chapter):
+        """Page count of chapter."""
+        if self.is_closed:
+            raise ValueError("document closed")
+        chapters = mupdf.fz_count_chapters( self.this)
+        if chapter < 0 or chapter >= chapters:
+            raise ValueError( "bad chapter number")
+        pages = mupdf.fz_count_chapter_pages( self.this, chapter)
+        return pages
+
+    def close(self):
+        """Close document."""
+        if getattr(self, "is_closed", True):
+            raise ValueError("document closed")
+        # self._cleanup()
+        if hasattr(self, "_outline") and self._outline:
+            self._outline = None
+        self._reset_page_refs()
+        #self.metadata    = None
+        #self.stream      = None
+        self.is_closed    = True
+        #self.FontInfos   = []
+        self.Graftmaps = {} # Fixes test_3140().
+        #self.ShownPages = {}
+        #self.InsertedImages  = {}
+        #self.this = None
+        self.this = None
+
+    def convert_to_pdf(self, from_page=0, to_page=-1, rotate=0):
+        """Convert document to a PDF, selecting page range and optional rotation. Output bytes object."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        fz_doc = self.this
+        fp = from_page
+        tp = to_page
+        srcCount = mupdf.fz_count_pages(fz_doc)
+        if fp < 0:
+            fp = 0
+        if fp > srcCount - 1:
+            fp = srcCount - 1
+        if tp < 0:
+            tp = srcCount - 1
+        if tp > srcCount - 1:
+            tp = srcCount - 1
+        len0 = len(JM_mupdf_warnings_store)
+        doc = JM_convert_to_pdf(fz_doc, fp, tp, rotate)
+        len1 = len(JM_mupdf_warnings_store)
+        for i in range(len0, len1):
+            message(f'{JM_mupdf_warnings_store[i]}')
+        return doc
+
+    def copy_page(self, pno: int, to: int =-1):
+        """Copy a page within a PDF document.
+
+        This will only create another reference of the same page object.
+        Args:
+            pno: source page number
+            to: put before this page, '-1' means after last page.
+        """
+        if self.is_closed:
+            raise ValueError("document closed")
+
+        page_count = len(self)
+        if (
+                pno not in range(page_count)
+                or to not in range(-1, page_count)
+                ):
+            raise ValueError("bad page number(s)")
+        before = 1
+        copy = 1
+        if to == -1:
+            to = page_count - 1
+            before = 0
+
+        return self._move_copy_page(pno, to, before, copy)
+
+    def del_xml_metadata(self):
+        """Delete XML metadata."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        root = mupdf.pdf_dict_get( mupdf.pdf_trailer( pdf), PDF_NAME('Root'))
+        if root.m_internal:
+            mupdf.pdf_dict_del( root, PDF_NAME('Metadata'))
+
+    def delete_page(self, pno: int =-1):
+        """ Delete one page from a PDF.
+        """
+        if not self.is_pdf:
+            raise ValueError("is no PDF")
+        if self.is_closed:
+            raise ValueError("document closed")
+
+        page_count = self.page_count
+        while pno < 0:
+            pno += page_count
+
+        if pno >= page_count:
+            raise ValueError("bad page number(s)")
+
+        # remove TOC bookmarks pointing to deleted page
+        toc = self.get_toc()
+        ol_xrefs = self.get_outline_xrefs()
+        for i, item in enumerate(toc):
+            if item[2] == pno + 1:
+                self._remove_toc_item(ol_xrefs[i])
+
+        self._remove_links_to(frozenset((pno,)))
+        self._delete_page(pno)
+        self._reset_page_refs()
+
+    def delete_pages(self, *args, **kw):
+        """Delete pages from a PDF.
+
+        Args:
+            Either keywords 'from_page'/'to_page', or two integers to
+            specify the first/last page to delete.
+            Or a list/tuple/range object, which can contain arbitrary
+            page numbers.
+        """
+        if not self.is_pdf:
+            raise ValueError("is no PDF")
+        if self.is_closed:
+            raise ValueError("document closed")
+
+        page_count = self.page_count  # page count of document
+        f = t = -1
+        if kw:  # check if keywords were used
+            if args:  # then no positional args are allowed
+                raise ValueError("cannot mix keyword and positional argument")
+            f = kw.get("from_page", -1)  # first page to delete
+            t = kw.get("to_page", -1)  # last page to delete
+            while f < 0:
+                f += page_count
+            while t < 0:
+                t += page_count
+            if not f <= t < page_count:
+                raise ValueError("bad page number(s)")
+            numbers = tuple(range(f, t + 1))
+        else:
+            if len(args) > 2 or args == []:
+                raise ValueError("need 1 or 2 positional arguments")
+            if len(args) == 2:
+                f, t = args
+                if not (type(f) is int and type(t) is int):
+                    raise ValueError("both arguments must be int")
+                if f > t:
+                    f, t = t, f
+                if not f <= t < page_count:
+                    raise ValueError("bad page number(s)")
+                numbers = tuple(range(f, t + 1))
+            else:
+                r = args[0]
+                if type(r) not in (int, range, list, tuple):
+                    raise ValueError("need int or sequence if one argument")
+                numbers = tuple(r)
+
+        numbers = list(map(int, set(numbers)))  # ensure unique integers
+        if numbers == []:
+            message("nothing to delete")
+            return
+        numbers.sort()
+        if numbers[0] < 0 or numbers[-1] >= page_count:
+            raise ValueError("bad page number(s)")
+        frozen_numbers = frozenset(numbers)
+        toc = self.get_toc()
+        for i, xref in enumerate(self.get_outline_xrefs()):
+            if toc[i][2] - 1 in frozen_numbers:
+                self._remove_toc_item(xref)  # remove target in PDF object
+
+        self._remove_links_to(frozen_numbers)
+
+        for i in reversed(numbers):  # delete pages, last to first
+            self._delete_page(i)
+
+        self._reset_page_refs()
+
+    def embfile_add(self,
+            name: str,
+            buffer_: ByteString,
+            filename: OptStr =None,
+            ufilename: OptStr =None,
+            desc: OptStr =None,
+            ) -> None:
+        """Add an item to the EmbeddedFiles array.
+
+        Args:
+            name: name of the new item, must not already exist.
+            buffer_: (binary data) the file content.
+            filename: (str) the file name, default: the name
+            ufilename: (unicode) the file name, default: filename
+            desc: (str) the description.
+        """
+        filenames = self.embfile_names()
+        msg = "Name '%s' already exists." % str(name)
+        if name in filenames:
+            raise ValueError(msg)
+
+        if filename is None:
+            filename = name
+        if ufilename is None:
+            ufilename = filename
+        if desc is None:
+            desc = name
+        xref = self._embfile_add(
+                name,
+                buffer_=buffer_,
+                filename=filename,
+                ufilename=ufilename,
+                desc=desc,
+                )
+        date = get_pdf_now()
+        self.xref_set_key(xref, "Type", "/EmbeddedFile")
+        self.xref_set_key(xref, "Params/CreationDate", get_pdf_str(date))
+        self.xref_set_key(xref, "Params/ModDate", get_pdf_str(date))
+        return xref
+
+    def embfile_count(self) -> int:
+        """Get number of EmbeddedFiles."""
+        return len(self.embfile_names())
+
+    def embfile_del(self, item: typing.Union[int, str]):
+        """Delete an entry from EmbeddedFiles.
+
+        Notes:
+            The argument must be name or index of an EmbeddedFiles item.
+            Physical deletion of data will happen on save to a new
+            file with appropriate garbage option.
+        Args:
+            item: name or number of item.
+        Returns:
+            None
+        """
+        idx = self._embeddedFileIndex(item)
+        return self._embfile_del(idx)
+
+    def embfile_get(self, item: typing.Union[int, str]) -> bytes:
+        """Get the content of an item in the EmbeddedFiles array.
+
+        Args:
+            item: number or name of item.
+        Returns:
+            (bytes) The file content.
+        """
+        idx = self._embeddedFileIndex(item)
+        return self._embeddedFileGet(idx)
+
+    def embfile_info(self, item: typing.Union[int, str]) -> dict:
+        """Get information of an item in the EmbeddedFiles array.
+
+        Args:
+            item: number or name of item.
+        Returns:
+            Information dictionary.
+        """
+        idx = self._embeddedFileIndex(item)
+        infodict = {"name": self.embfile_names()[idx]}
+        xref = self._embfile_info(idx, infodict)
+        t, date = self.xref_get_key(xref, "Params/CreationDate")
+        if t != "null":
+            infodict["creationDate"] = date
+        t, date = self.xref_get_key(xref, "Params/ModDate")
+        if t != "null":
+            infodict["modDate"] = date
+        t, md5 = self.xref_get_key(xref, "Params/CheckSum")
+        if t != "null":
+            infodict["checksum"] = binascii.hexlify(md5.encode()).decode()
+        return infodict
+
+    def embfile_names(self) -> list:
+        """Get list of names of EmbeddedFiles."""
+        filenames = []
+        self._embfile_names(filenames)
+        return filenames
+
+    def embfile_upd(self,
+            item: typing.Union[int, str],
+            buffer_: OptBytes =None,
+            filename: OptStr =None,
+            ufilename: OptStr =None,
+            desc: OptStr =None,
+            ) -> None:
+        """Change an item of the EmbeddedFiles array.
+
+        Notes:
+            Only provided parameters are changed. If all are omitted,
+            the method is a no-op.
+        Args:
+            item: number or name of item.
+            buffer_: (binary data) the new file content.
+            filename: (str) the new file name.
+            ufilename: (unicode) the new filen ame.
+            desc: (str) the new description.
+        """
+        idx = self._embeddedFileIndex(item)
+        xref = self._embfile_upd(
+                idx,
+                buffer_=buffer_,
+                filename=filename,
+                ufilename=ufilename,
+                desc=desc,
+                )
+        date = get_pdf_now()
+        self.xref_set_key(xref, "Params/ModDate", get_pdf_str(date))
+        return xref
+
+    def extract_font(self, xref=0, info_only=0, named=None):
+        '''
+        Get a font by xref. Returns a tuple or dictionary.
+        '''
+        #log( '{=xref info_only}')
+        pdf = _as_pdf_document(self)
+        obj = mupdf.pdf_load_object(pdf, xref)
+        type_ = mupdf.pdf_dict_get(obj, PDF_NAME('Type'))
+        subtype = mupdf.pdf_dict_get(obj, PDF_NAME('Subtype'))
+        if (mupdf.pdf_name_eq(type_, PDF_NAME('Font'))
+                and not mupdf.pdf_to_name( subtype).startswith('CIDFontType')
+                ):
+            basefont = mupdf.pdf_dict_get(obj, PDF_NAME('BaseFont'))
+            if not basefont.m_internal or mupdf.pdf_is_null(basefont):
+                bname = mupdf.pdf_dict_get(obj, PDF_NAME('Name'))
+            else:
+                bname = basefont
+            ext = JM_get_fontextension(pdf, xref)
+            if ext != 'n/a' and not info_only:
+                buffer_ = JM_get_fontbuffer(pdf, xref)
+                bytes_ = JM_BinFromBuffer(buffer_)
+            else:
+                bytes_ = b''
+            if not named:
+                rc = (
+                        JM_EscapeStrFromStr(mupdf.pdf_to_name(bname)),
+                        JM_UnicodeFromStr(ext),
+                        JM_UnicodeFromStr(mupdf.pdf_to_name(subtype)),
+                        bytes_,
+                        )
+            else:
+                rc = {
+                        dictkey_name: JM_EscapeStrFromStr(mupdf.pdf_to_name(bname)),
+                        dictkey_ext: JM_UnicodeFromStr(ext),
+                        dictkey_type: JM_UnicodeFromStr(mupdf.pdf_to_name(subtype)),
+                        dictkey_content: bytes_,
+                        }
+        else:
+            if not named:
+                rc = '', '', '', b''
+            else:
+                rc = {
+                        dictkey_name: '',
+                        dictkey_ext: '',
+                        dictkey_type: '',
+                        dictkey_content: b'',
+                        }
+        return rc
+
+    def extract_image(self, xref):
+        """Get image by xref. Returns a dictionary."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+
+        pdf = _as_pdf_document(self)
+
+        if not _INRANGE(xref, 1, mupdf.pdf_xref_len(pdf)-1):
+            raise ValueError( MSG_BAD_XREF)
+
+        obj = mupdf.pdf_new_indirect(pdf, xref, 0)
+        subtype = mupdf.pdf_dict_get(obj, PDF_NAME('Subtype'))
+
+        if not mupdf.pdf_name_eq(subtype, PDF_NAME('Image')):
+            raise ValueError( "not an image")
+
+        o = mupdf.pdf_dict_geta(obj, PDF_NAME('SMask'), PDF_NAME('Mask'))
+        if o.m_internal:
+            smask = mupdf.pdf_to_num(o)
+        else:
+            smask = 0
+
+        # load the image
+        img = mupdf.pdf_load_image(pdf, obj)
+        rc = dict()
+        _make_image_dict(img, rc)
+        rc[dictkey_smask] = smask
+        rc[dictkey_cs_name] = mupdf.fz_colorspace_name(img.colorspace())
+        return rc
+
+    def ez_save(
+            self,
+            filename,
+            garbage=3,
+            clean=False,
+            deflate=True,
+            deflate_images=True,
+            deflate_fonts=True,
+            incremental=False,
+            ascii=False,
+            expand=False,
+            linear=False,
+            pretty=False,
+            encryption=1,
+            permissions=4095,
+            owner_pw=None,
+            user_pw=None,
+            no_new_id=True,
+            preserve_metadata=1,
+            use_objstms=1,
+            compression_effort=0,
+            ):
+        '''
+        Save PDF using some different defaults
+        '''
+        return self.save(
+                filename,
+                garbage=garbage,
+                clean=clean,
+                deflate=deflate,
+                deflate_images=deflate_images,
+                deflate_fonts=deflate_fonts,
+                incremental=incremental,
+                ascii=ascii,
+                expand=expand,
+                linear=linear,
+                pretty=pretty,
+                encryption=encryption,
+                permissions=permissions,
+                owner_pw=owner_pw,
+                user_pw=user_pw,
+                no_new_id=no_new_id,
+                preserve_metadata=preserve_metadata,
+                use_objstms=use_objstms,
+                compression_effort=compression_effort,
+                )
+
+    def find_bookmark(self, bm):
+        """Find new location after layouting a document."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        location = mupdf.fz_lookup_bookmark2( self.this, bm)
+        return location.chapter, location.page
+
+    def fullcopy_page(self, pno, to=-1):
+        """Make a full page duplicate."""
+        pdf = _as_pdf_document(self)
+        page_count = mupdf.pdf_count_pages( pdf)
+        try:
+            if (not _INRANGE(pno, 0, page_count - 1)
+                    or not _INRANGE(to, -1, page_count - 1)
+                    ):
+                raise ValueError( MSG_BAD_PAGENO)
+
+            page1 = mupdf.pdf_resolve_indirect( mupdf.pdf_lookup_page_obj( pdf, pno))
+
+            page2 = mupdf.pdf_deep_copy_obj( page1)
+            old_annots = mupdf.pdf_dict_get( page2, PDF_NAME('Annots'))
+
+            # copy annotations, but remove Popup and IRT types
+            if old_annots.m_internal:
+                n = mupdf.pdf_array_len( old_annots)
+                new_annots = mupdf.pdf_new_array( pdf, n)
+                for i in range(n):
+                    o = mupdf.pdf_array_get( old_annots, i)
+                    subtype = mupdf.pdf_dict_get( o, PDF_NAME('Subtype'))
+                    if mupdf.pdf_name_eq( subtype, PDF_NAME('Popup')):
+                        continue
+                    if mupdf.pdf_dict_gets( o, "IRT").m_internal:
+                        continue
+                    copy_o = mupdf.pdf_deep_copy_obj( mupdf.pdf_resolve_indirect( o))
+                    xref = mupdf.pdf_create_object( pdf)
+                    mupdf.pdf_update_object( pdf, xref, copy_o)
+                    copy_o = mupdf.pdf_new_indirect( pdf, xref, 0)
+                    mupdf.pdf_dict_del( copy_o, PDF_NAME('Popup'))
+                    mupdf.pdf_dict_del( copy_o, PDF_NAME('P'))
+                    mupdf.pdf_array_push( new_annots, copy_o)
+                mupdf.pdf_dict_put( page2, PDF_NAME('Annots'), new_annots)
+
+            # copy the old contents stream(s)
+            res = JM_read_contents( page1)
+
+            # create new /Contents object for page2
+            if res and res.m_internal:
+                #contents = mupdf.pdf_add_stream( pdf, mupdf.fz_new_buffer_from_copied_data( b"  ", 1), NULL, 0)
+                contents = mupdf.pdf_add_stream( pdf, mupdf.fz_new_buffer_from_copied_data( b" "), mupdf.PdfObj(), 0)
+                JM_update_stream( pdf, contents, res, 1)
+                mupdf.pdf_dict_put( page2, PDF_NAME('Contents'), contents)
+
+            # now insert target page, making sure it is an indirect object
+            xref = mupdf.pdf_create_object( pdf)   # get new xref
+            mupdf.pdf_update_object( pdf, xref, page2) # store new page
+
+            page2 = mupdf.pdf_new_indirect( pdf, xref, 0)  # reread object
+            mupdf.pdf_insert_page( pdf, to, page2) # and store the page
+        finally:
+            mupdf.ll_pdf_drop_page_tree( pdf.m_internal)
+
+        self._reset_page_refs()
+
+    def get_layer(self, config=-1):
+        """Content of ON, OFF, RBGroups of an OC layer."""
+        pdf = _as_pdf_document(self)
+        ocp = mupdf.pdf_dict_getl(
+                mupdf.pdf_trailer( pdf),
+                PDF_NAME('Root'),
+                PDF_NAME('OCProperties'),
+                )
+        if not ocp.m_internal:
+            return
+        if config == -1:
+            obj = mupdf.pdf_dict_get( ocp, PDF_NAME('D'))
+        else:
+            obj = mupdf.pdf_array_get(
+                    mupdf.pdf_dict_get( ocp, PDF_NAME('Configs')),
+                    config,
+                    )
+        if not obj.m_internal:
+            raise ValueError( MSG_BAD_OC_CONFIG)
+        rc = JM_get_ocg_arrays( obj)
+        return rc
+
+    def get_layers(self):
+        """Show optional OC layers."""
+        pdf = _as_pdf_document(self)
+        n = mupdf.pdf_count_layer_configs( pdf)
+        if n == 1:
+            obj = mupdf.pdf_dict_getl(
+                    mupdf.pdf_trailer( pdf),
+                    PDF_NAME('Root'),
+                    PDF_NAME('OCProperties'),
+                    PDF_NAME('Configs'),
+                    )
+            if not mupdf.pdf_is_array( obj):
+                n = 0
+        rc = []
+        info = mupdf.PdfLayerConfig()
+        for i in range(n):
+            mupdf.pdf_layer_config_info( pdf, i, info)
+            item = {
+                    "number": i,
+                    "name": info.name,
+                    "creator": info.creator,
+                    }
+            rc.append( item)
+        return rc
+
+    def get_new_xref(self):
+        """Make new xref."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        xref = 0
+        ENSURE_OPERATION(pdf)
+        xref = mupdf.pdf_create_object(pdf)
+        return xref
+
+    def get_ocgs(self):
+        """Show existing optional content groups."""
+        ci = mupdf.pdf_new_name( "CreatorInfo")
+        pdf = _as_pdf_document(self)
+        ocgs = mupdf.pdf_dict_getl(
+                mupdf.pdf_dict_get( mupdf.pdf_trailer( pdf), PDF_NAME('Root')),
+                PDF_NAME('OCProperties'),
+                PDF_NAME('OCGs'),
+                )
+        rc = dict()
+        if not mupdf.pdf_is_array( ocgs):
+            return rc
+        n = mupdf.pdf_array_len( ocgs)
+        for i in range(n):
+            ocg = mupdf.pdf_array_get( ocgs, i)
+            xref = mupdf.pdf_to_num( ocg)
+            name = mupdf.pdf_to_text_string( mupdf.pdf_dict_get( ocg, PDF_NAME('Name')))
+            obj = mupdf.pdf_dict_getl( ocg, PDF_NAME('Usage'), ci, PDF_NAME('Subtype'))
+            usage = None
+            if obj.m_internal:
+                usage = mupdf.pdf_to_name( obj)
+            intents = list()
+            intent = mupdf.pdf_dict_get( ocg, PDF_NAME('Intent'))
+            if intent.m_internal:
+                if mupdf.pdf_is_name( intent):
+                    intents.append( mupdf.pdf_to_name( intent))
+                elif mupdf.pdf_is_array( intent):
+                    m = mupdf.pdf_array_len( intent)
+                    for j in range(m):
+                        o = mupdf.pdf_array_get( intent, j)
+                        if mupdf.pdf_is_name( o):
+                            intents.append( mupdf.pdf_to_name( o))
+            hidden = mupdf.pdf_is_ocg_hidden( pdf, mupdf.PdfObj(), usage, ocg)
+            item = {
+                    "name": name,
+                    "intent": intents,
+                    "on": not hidden,
+                    "usage": usage,
+                    }
+            temp = xref
+            rc[ temp] = item
+        return rc
+
+    def get_outline_xrefs(self):
+        """Get list of outline xref numbers."""
+        xrefs = []
+        pdf = _as_pdf_document(self, required=0)
+        if not pdf.m_internal:
+            return xrefs
+        root = mupdf.pdf_dict_get(mupdf.pdf_trailer(pdf), PDF_NAME('Root'))
+        if not root.m_internal:
+            return xrefs
+        olroot = mupdf.pdf_dict_get(root, PDF_NAME('Outlines'))
+        if not olroot.m_internal:
+            return xrefs
+        first = mupdf.pdf_dict_get(olroot, PDF_NAME('First'))
+        if not first.m_internal:
+            return xrefs
+        xrefs = JM_outline_xrefs(first, xrefs)
+        return xrefs
+
+    def get_page_fonts(self, pno: int, full: bool =False) -> list:
+        """Retrieve a list of fonts used on a page.
+        """
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        if not self.is_pdf:
+            return ()
+        if type(pno) is not int:
+            try:
+                pno = pno.number
+            except Exception:
+                exception_info()
+                raise ValueError("need a Page or page number")
+        val = self._getPageInfo(pno, 1)
+        if not full:
+            return [v[:-1] for v in val]
+        return val
+
+    def get_page_images(self, pno: int, full: bool =False) -> list:
+        """Retrieve a list of images used on a page.
+        """
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        if not self.is_pdf:
+            return ()
+        val = self._getPageInfo(pno, 2)
+        if not full:
+            return [v[:-1] for v in val]
+        return val
+
+    def get_page_xobjects(self, pno: int) -> list:
+        """Retrieve a list of XObjects used on a page.
+        """
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        if not self.is_pdf:
+            return ()
+        val = self._getPageInfo(pno, 3)
+        return val
+
+    def get_sigflags(self):
+        """Get the /SigFlags value."""
+        pdf = _as_pdf_document(self, required=0)
+        if not pdf.m_internal:
+            return -1   # not a PDF
+        sigflags = mupdf.pdf_dict_getl(
+                mupdf.pdf_trailer(pdf),
+                PDF_NAME('Root'),
+                PDF_NAME('AcroForm'),
+                PDF_NAME('SigFlags'),
+                )
+        sigflag = -1
+        if sigflags.m_internal:
+            sigflag = mupdf.pdf_to_int(sigflags)
+        return sigflag
+
+    def get_xml_metadata(self):
+        """Get document XML metadata."""
+        xml = None
+        pdf = _as_pdf_document(self, required=0)
+        if pdf.m_internal:
+            xml = mupdf.pdf_dict_getl(
+                    mupdf.pdf_trailer(pdf),
+                    PDF_NAME('Root'),
+                    PDF_NAME('Metadata'),
+                    )
+        if xml is not None and xml.m_internal:
+            buff = mupdf.pdf_load_stream(xml)
+            rc = JM_UnicodeFromBuffer(buff)
+        else:
+            rc = ''
+        return rc
+
+    def init_doc(self):
+        if self.is_encrypted:
+            raise ValueError("cannot initialize - document still encrypted")
+        self._outline = self._loadOutline()
+        self.metadata = dict(
+                    [
+                        (k,self._getMetadata(v)) for k,v in {
+                            'format':'format',
+                            'title':'info:Title',
+                            'author':'info:Author',
+                            'subject':'info:Subject',
+                            'keywords':'info:Keywords',
+                            'creator':'info:Creator',
+                            'producer':'info:Producer',
+                            'creationDate':'info:CreationDate',
+                            'modDate':'info:ModDate',
+                            'trapped':'info:Trapped'
+                            }.items()
+                    ]
+                )
+        self.metadata['encryption'] = None if self._getMetadata('encryption')=='None' else self._getMetadata('encryption')
+
+    def insert_file(self,
+            infile,
+            from_page=-1,
+            to_page=-1,
+            start_at=-1,
+            rotate=-1,
+            links=True,
+            annots=True,
+            show_progress=0,
+            final=1,
+            ):
+        '''
+        Insert an arbitrary supported document to an existing PDF.
+
+        The infile may be given as a filename, a Document or a Pixmap. Other
+        parameters - where applicable - equal those of insert_pdf().
+        '''
+        src = None
+        if isinstance(infile, Pixmap):
+            if infile.colorspace.n > 3:
+                infile = Pixmap(csRGB, infile)
+            src = Document("png", infile.tobytes())
+        elif isinstance(infile, Document):
+            src = infile
+        else:
+            src = Document(infile)
+        if not src:
+            raise ValueError("bad infile parameter")
+        if not src.is_pdf:
+            pdfbytes = src.convert_to_pdf()
+            src = Document("pdf", pdfbytes)
+        return self.insert_pdf(
+                src,
+                from_page=from_page,
+                to_page=to_page,
+                start_at=start_at,
+                rotate=rotate,
+                links=links,
+                annots=annots,
+                show_progress=show_progress,
+                final=final,
+                )
+
+    def insert_pdf(
+            self,
+            docsrc,
+            *,
+            from_page=-1,
+            to_page=-1,
+            start_at=-1,
+            rotate=-1,
+            links=1,
+            annots=1,
+            widgets=1,
+            join_duplicates=0,
+            show_progress=0,
+            final=1,
+            _gmap=None,
+            ):
+        """Insert a page range from another PDF.
+
+        Args:
+            docsrc: PDF to copy from. Must be different object, but may be same file.
+            from_page: (int) first source page to copy, 0-based, default 0.
+            to_page: (int) last source page to copy, 0-based, default last page.
+            start_at: (int) from_page will become this page number in target.
+            rotate: (int) rotate copied pages, default -1 is no change.
+            links: (int/bool) whether to also copy links.
+            annots: (int/bool) whether to also copy annotations.
+            widgets: (int/bool) whether to also copy form fields.
+            join_duplicates: (int/bool) join or rename duplicate widget names.
+            show_progress: (int) progress message interval, 0 is no messages.
+            final: (bool) indicates last insertion from this source PDF.
+            _gmap: internal use only
+
+        Copy sequence reversed if from_page > to_page."""
+
+        # Insert pages from a source PDF into this PDF.
+        # For reconstructing the links (_do_links method), we must save the
+        # insertion point (start_at) if it was specified as -1.
+        #log( 'insert_pdf(): start')
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        if self._graft_id == docsrc._graft_id:
+            raise ValueError("source and target cannot be same object")
+        sa = start_at
+        if sa < 0:
+            sa = self.page_count
+        outCount = self.page_count
+        srcCount = docsrc.page_count
+
+        # local copies of page numbers
+        fp = from_page
+        tp = to_page
+        sa = start_at
+
+        # normalize page numbers
+        fp = max(fp, 0) # -1 = first page
+        fp = min(fp, srcCount - 1)  # but do not exceed last page
+
+        if tp < 0:
+            tp = srcCount - 1   # -1 = last page
+        tp = min(tp, srcCount - 1)  # but do not exceed last page
+
+        if sa < 0:
+            sa = outCount   # -1 = behind last page
+        sa = min(sa, outCount)  # but that is also the limit
+
+        if len(docsrc) > show_progress > 0:
+            inname = os.path.basename(docsrc.name)
+            if not inname:
+                inname = "memory PDF"
+            outname = os.path.basename(self.name)
+            if not outname:
+                outname = "memory PDF"
+            message("Inserting '%s' at '%s'" % (inname, outname))
+
+        # retrieve / make a Graftmap to avoid duplicate objects
+        #log( 'insert_pdf(): Graftmaps')
+        isrt = docsrc._graft_id
+        _gmap = self.Graftmaps.get(isrt, None)
+        if _gmap is None:
+            #log( 'insert_pdf(): Graftmaps2')
+            _gmap = Graftmap(self)
+            self.Graftmaps[isrt] = _gmap
+
+        if g_use_extra:
+            #log( 'insert_pdf(): calling extra_FzDocument_insert_pdf()')
+            extra_FzDocument_insert_pdf(
+                    self.this,
+                    docsrc.this,
+                    from_page,
+                    to_page,
+                    start_at,
+                    rotate,
+                    links,
+                    annots,
+                    show_progress,
+                    final,
+                    _gmap,
+                    )
+            #log( 'insert_pdf(): extra_FzDocument_insert_pdf() returned.')
+        else:
+            pdfout = _as_pdf_document(self)
+            pdfsrc = _as_pdf_document(docsrc)
+
+            if not pdfout.m_internal or not pdfsrc.m_internal:
+                raise TypeError( "source or target not a PDF")
+            ENSURE_OPERATION(pdfout)
+            JM_merge_range(pdfout, pdfsrc, fp, tp, sa, rotate, links, annots, show_progress, _gmap)
+        
+        #log( 'insert_pdf(): calling self._reset_page_refs()')
+        self._reset_page_refs()
+        if links:
+            #log( 'insert_pdf(): calling self._do_links()')
+            self._do_links(docsrc, from_page=fp, to_page=tp, start_at=sa)
+        if widgets:
+            self._do_widgets(docsrc, _gmap, from_page=fp, to_page=tp, start_at=sa, join_duplicates=join_duplicates)
+        if final == 1:
+            self.Graftmaps[isrt] = None
+        #log( 'insert_pdf(): returning')
+
+    @property
+    def is_dirty(self):
+        pdf = _as_pdf_document(self, required=0)
+        if not pdf.m_internal:
+            return False
+        r = mupdf.pdf_has_unsaved_changes(pdf)
+        return True if r else False
+
+    @property
+    def is_fast_webaccess(self):
+        '''
+        Check whether we have a linearized PDF.
+        '''
+        pdf = _as_pdf_document(self, required=0)
+        if pdf.m_internal:
+            return mupdf.pdf_doc_was_linearized(pdf)
+        return False    # gracefully handle non-PDF
+
+    @property
+    def is_form_pdf(self):
+        """Either False or PDF field count."""
+        pdf = _as_pdf_document(self, required=0)
+        if not pdf.m_internal:
+            return False
+        count = -1
+        try:
+            fields = mupdf.pdf_dict_getl(
+                    mupdf.pdf_trailer(pdf),
+                    mupdf.PDF_ENUM_NAME_Root,
+                    mupdf.PDF_ENUM_NAME_AcroForm,
+                    mupdf.PDF_ENUM_NAME_Fields,
+                    )
+            if mupdf.pdf_is_array(fields):
+                count = mupdf.pdf_array_len(fields)
+        except Exception:
+            if g_exceptions_verbose:    exception_info()
+            return False
+        if count >= 0:
+            return count
+        return False
+
+    @property
+    def is_pdf(self):
+        """Check for PDF."""
+        if isinstance(self.this, mupdf.PdfDocument):
+            return True
+        # Avoid calling smupdf.pdf_specifics because it will end up creating
+        # a new PdfDocument which will call pdf_create_document(), which is ok
+        # but a little unnecessary.
+        #
+        if mupdf.ll_pdf_specifics(self.this.m_internal):
+            ret = True
+        else:
+            ret = False
+        return ret
+
+    @property
+    def is_reflowable(self):
+        """Check if document is layoutable."""
+        if self.is_closed:
+            raise ValueError("document closed")
+        return bool(mupdf.fz_is_document_reflowable(self))
+
+    @property
+    def is_repaired(self):
+        """Check whether PDF was repaired."""
+        pdf = _as_pdf_document(self, required=0)
+        if not pdf.m_internal:
+            return False
+        r = mupdf.pdf_was_repaired(pdf)
+        if r:
+            return True
+        return False
+
+    def journal_can_do(self):
+        """Show if undo and / or redo are possible."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        undo=0
+        redo=0
+        pdf = _as_pdf_document(self)
+        undo = mupdf.pdf_can_undo(pdf)
+        redo = mupdf.pdf_can_redo(pdf)
+        return {'undo': bool(undo), 'redo': bool(redo)}
+
+    def journal_enable(self):
+        """Activate document journalling."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        mupdf.pdf_enable_journal(pdf)
+
+    def journal_is_enabled(self):
+        """Check if journalling is enabled."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        enabled = pdf.m_internal and pdf.m_internal.journal
+        return enabled
+
+    def journal_load(self, filename):
+        """Load a journal from a file."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        if isinstance(filename, str):
+            mupdf.pdf_load_journal(pdf, filename)
+        else:
+            res = JM_BufferFromBytes(filename)
+            stm = mupdf.fz_open_buffer(res)
+            mupdf.pdf_deserialise_journal(pdf, stm)
+        if not pdf.m_internal.journal:
+            RAISEPY( "Journal and document do not match", JM_Exc_FileDataError)
+
+    def journal_op_name(self, step):
+        """Show operation name for given step."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        name = mupdf.pdf_undoredo_step(pdf, step)
+        return name
+
+    def journal_position(self):
+        """Show journalling state."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        steps=0
+        pdf = _as_pdf_document(self)
+        rc, steps = mupdf.pdf_undoredo_state(pdf)
+        return rc, steps
+
+    def journal_redo(self):
+        """Move forward in the journal."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        mupdf.pdf_redo(pdf)
+        return True
+
+    def journal_save(self, filename):
+        """Save journal to a file."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        if isinstance(filename, str):
+            mupdf.pdf_save_journal(pdf, filename)
+        else:
+            out = JM_new_output_fileptr(filename)
+            mupdf.pdf_write_journal(pdf, out)
+            out.fz_close_output()
+
+    def journal_start_op(self, name=None):
+        """Begin a journalling operation."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        if not pdf.m_internal.journal:
+            raise RuntimeError( "Journalling not enabled")
+        if name:
+            mupdf.pdf_begin_operation(pdf, name)
+        else:
+            mupdf.pdf_begin_implicit_operation(pdf)
+
+    def journal_stop_op(self):
+        """End a journalling operation."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        mupdf.pdf_end_operation(pdf)
+
+    def journal_undo(self):
+        """Move backwards in the journal."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        mupdf.pdf_undo(pdf)
+        return True
+
+    @property
+    def language(self):
+        """Document language."""
+        pdf = _as_pdf_document(self, required=0)
+        if not pdf.m_internal:
+            return
+        lang = mupdf.pdf_document_language(pdf)
+        if lang == mupdf.FZ_LANG_UNSET:
+            return
+        return mupdf.fz_string_from_text_language2(lang)
+
+    @property
+    def last_location(self):
+        """Id (chapter, page) of last page."""
+        if self.is_closed:
+            raise ValueError("document closed")
+        last_loc = mupdf.fz_last_page(self.this)
+        return last_loc.chapter, last_loc.page
+
+    def layer_ui_configs(self):
+        """Show OC visibility status modifiable by user."""
+        pdf = _as_pdf_document(self)
+        info = mupdf.PdfLayerConfigUi()
+        n = mupdf.pdf_count_layer_config_ui( pdf)
+        rc = []
+        for i in range(n):
+            mupdf.pdf_layer_config_ui_info( pdf, i, info)
+            if info.type == 1:
+                type_ = "checkbox"
+            elif info.type == 2:
+                type_ = "radiobox"
+            else:
+                type_ = "label"
+            item = {
+                    "number": i,
+                    "text": info.text,
+                    "depth": info.depth,
+                    "type": type_,
+                    "on": info.selected,
+                    "locked": info.locked,
+                    }
+            rc.append(item)
+        return rc
+
+    def layout(self, rect=None, width=0, height=0, fontsize=11):
+        """Re-layout a reflowable document."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        doc = self.this
+        if not mupdf.fz_is_document_reflowable( doc):
+            return
+        w = width
+        h = height
+        r = JM_rect_from_py(rect)
+        if not mupdf.fz_is_infinite_rect(r):
+            w = r.x1 - r.x0
+            h = r.y1 - r.y0
+        if w <= 0.0 or h <= 0.0:
+            raise ValueError( "bad page size")
+        mupdf.fz_layout_document( doc, w, h, fontsize)
+
+        self._reset_page_refs()
+        self.init_doc()
+
+    def load_page(self, page_id):
+        """Load a page.
+
+        'page_id' is either a 0-based page number or a tuple (chapter, pno),
+        with chapter number and page number within that chapter.
+        """
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        if page_id is None:
+            page_id = 0
+        if page_id not in self:
+            raise ValueError("page not in document")
+        if type(page_id) is int and page_id < 0:
+            np = self.page_count
+            while page_id < 0:
+                page_id += np
+        if isinstance(page_id, int):
+            page = mupdf.fz_load_page(self.this, page_id)
+        else:
+            chapter, pagenum = page_id
+            page = mupdf.fz_load_chapter_page(self.this, chapter, pagenum)
+        val = Page(page, self)
+
+        val.thisown = True
+        val.parent = self
+        self._page_refs[id(val)] = val
+        val._annot_refs = weakref.WeakValueDictionary()
+        val.number = page_id
+        return val
+
+    def location_from_page_number(self, pno):
+        """Convert pno to (chapter, page)."""
+        if self.is_closed:
+            raise ValueError("document closed")
+        this_doc = self.this
+        loc = mupdf.fz_make_location(-1, -1)
+        page_count = mupdf.fz_count_pages(this_doc)
+        while pno < 0:
+            pno += page_count
+        if pno >= page_count:
+            raise ValueError( MSG_BAD_PAGENO)
+        loc = mupdf.fz_location_from_page_number(this_doc, pno)
+        return loc.chapter, loc.page
+
+    def make_bookmark(self, loc):
+        """Make a page pointer before layouting document."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        loc = mupdf.FzLocation(*loc)
+        mark = mupdf.ll_fz_make_bookmark2( self.this.m_internal, loc.internal())
+        return mark
+
+    @property
+    def markinfo(self) -> dict:
+        """Return the PDF MarkInfo value."""
+        xref = self.pdf_catalog()
+        if xref == 0:
+            return None
+        rc = self.xref_get_key(xref, "MarkInfo")
+        if rc[0] == "null":
+            return {}
+        if rc[0] == "xref":
+            xref = int(rc[1].split()[0])
+            val = self.xref_object(xref, compressed=True)
+        elif rc[0] == "dict":
+            val = rc[1]
+        else:
+            val = None
+        if val is None or not (val[:2] == "<<" and val[-2:] == ">>"):
+            return {}
+        valid = {"Marked": False, "UserProperties": False, "Suspects": False}
+        val = val[2:-2].split("/")
+        for v in val[1:]:
+            try:
+                key, value = v.split()
+            except Exception:
+                if g_exceptions_verbose > 1:    exception_info()
+                return valid
+            if value == "true":
+                valid[key] = True
+        return valid
+
+    def move_page(self, pno: int, to: int =-1):
+        """Move a page within a PDF document.
+
+        Args:
+            pno: source page number.
+            to: put before this page, '-1' means after last page.
+        """
+        if self.is_closed:
+            raise ValueError("document closed")
+        page_count = len(self)
+        if (pno not in range(page_count) or to not in range(-1, page_count)):
+            raise ValueError("bad page number(s)")
+        before = 1
+        copy = 0
+        if to == -1:
+            to = page_count - 1
+            before = 0
+
+        return self._move_copy_page(pno, to, before, copy)
+
+    @property
+    def name(self):
+        return self._name
+    
+    def need_appearances(self, value=None):
+        """Get/set the NeedAppearances value."""
+        if not self.is_form_pdf:
+            return None
+        
+        pdf = _as_pdf_document(self)
+        oldval = -1
+        appkey = "NeedAppearances"
+        
+        form = mupdf.pdf_dict_getp(
+                mupdf.pdf_trailer(pdf),
+                "Root/AcroForm",
+                )
+        app = mupdf.pdf_dict_gets(form, appkey)
+        if mupdf.pdf_is_bool(app):
+            oldval = mupdf.pdf_to_bool(app)
+        if value:
+            mupdf.pdf_dict_puts(form, appkey, mupdf.PDF_TRUE)
+        else:
+            mupdf.pdf_dict_puts(form, appkey, mupdf.PDF_FALSE)
+        if value is None:
+            return oldval >= 0
+        return value
+
+    @property
+    def needs_pass(self):
+        """Indicate password required."""
+        if self.is_closed:
+            raise ValueError("document closed")
+        document = self.this if isinstance(self.this, mupdf.FzDocument) else self.this.super()
+        ret = mupdf.fz_needs_password( document)
+        return ret
+
+    def next_location(self, page_id):
+        """Get (chapter, page) of next page."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        if type(page_id) is int:
+            page_id = (0, page_id)
+        if page_id not in self:
+            raise ValueError("page id not in document")
+        if tuple(page_id) == self.last_location:
+            return ()
+        this_doc = _as_fz_document(self)
+        val = page_id[ 0]
+        if not isinstance(val, int):
+            RAISEPY(MSG_BAD_PAGEID, PyExc_ValueError)
+        chapter = val
+        val = page_id[ 1]
+        pno = val
+        loc = mupdf.fz_make_location(chapter, pno)
+        next_loc = mupdf.fz_next_page( this_doc, loc)
+        return next_loc.chapter, next_loc.page
+
+    def page_annot_xrefs(self, n):
+        if g_use_extra:
+            return extra.page_annot_xrefs( self.this, n)
+        
+        if isinstance(self.this, mupdf.PdfDocument):
+            page_count = mupdf.pdf_count_pages(self.this)
+            pdf_document = self.this
+        else:
+            page_count = mupdf.fz_count_pages(self.this)
+            pdf_document = _as_pdf_document(self)
+        while n < 0:
+            n += page_count
+        if n > page_count:
+            raise ValueError( MSG_BAD_PAGENO)
+        page_obj = mupdf.pdf_lookup_page_obj(pdf_document, n)
+        annots = JM_get_annot_xref_list(page_obj)
+        return annots
+
+    @property
+    def page_count(self):
+        """Number of pages."""
+        if self.is_closed:
+            raise ValueError('document closed')
+        if g_use_extra:
+            return self.page_count2(self)
+        if isinstance( self.this, mupdf.FzDocument):
+            return mupdf.fz_count_pages( self.this)
+        else:
+            return mupdf.pdf_count_pages( self.this)
+
+    def page_cropbox(self, pno):
+        """Get CropBox of page number (without loading page)."""
+        if self.is_closed:
+            raise ValueError("document closed")
+        this_doc = self.this
+        page_count = mupdf.fz_count_pages( this_doc)
+        n = pno
+        while n < 0:
+            n += page_count
+        pdf = _as_pdf_document(self)
+        if n >= page_count:
+            raise ValueError( MSG_BAD_PAGENO)
+        pageref = mupdf.pdf_lookup_page_obj( pdf, n)
+        cropbox = JM_cropbox(pageref)
+        val = JM_py_from_rect(cropbox)
+
+        val = Rect(val)
+
+        return val
+
+    def page_number_from_location(self, page_id):
+        """Convert (chapter, pno) to page number."""
+        if type(page_id) is int:
+            np = self.page_count
+            while page_id < 0:
+                page_id += np
+            page_id = (0, page_id)
+        if page_id not in self:
+            raise ValueError("page id not in document")
+        chapter, pno = page_id
+        loc = mupdf.fz_make_location( chapter, pno)
+        page_n = mupdf.fz_page_number_from_location( self.this, loc)
+        return page_n
+
+    def page_xref(self, pno):
+        """Get xref of page number."""
+        if g_use_extra:
+            return extra.page_xref( self.this, pno)
+        if self.is_closed:
+            raise ValueError("document closed")
+        page_count = mupdf.fz_count_pages(self.this)
+        n = pno
+        while n < 0:
+            n += page_count
+        pdf = _as_pdf_document(self)
+        xref = 0
+        if n >= page_count:
+            raise ValueError( MSG_BAD_PAGENO)
+        xref = mupdf.pdf_to_num(mupdf.pdf_lookup_page_obj(pdf, n))
+        return xref
+
+    @property
+    def pagelayout(self) -> str:
+        """Return the PDF PageLayout value.
+        """
+        xref = self.pdf_catalog()
+        if xref == 0:
+            return None
+        rc = self.xref_get_key(xref, "PageLayout")
+        if rc[0] == "null":
+            return "SinglePage"
+        if rc[0] == "name":
+            return rc[1][1:]
+        return "SinglePage"
+
+    @property
+    def pagemode(self) -> str:
+        """Return the PDF PageMode value.
+        """
+        xref = self.pdf_catalog()
+        if xref == 0:
+            return None
+        rc = self.xref_get_key(xref, "PageMode")
+        if rc[0] == "null":
+            return "UseNone"
+        if rc[0] == "name":
+            return rc[1][1:]
+        return "UseNone"
+
+    if sys.implementation.version < (3, 9):
+        # Appending `[Page]` causes `TypeError: 'ABCMeta' object is not subscriptable`.
+        _pages_ret = collections.abc.Iterable
+    else:
+        _pages_ret = collections.abc.Iterable[Page]
+
+    def pages(self, start: OptInt =None, stop: OptInt =None, step: OptInt =None) -> _pages_ret:
+        """Return a generator iterator over a page range.
+
+        Arguments have the same meaning as for the range() built-in.
+        """
+        if not self.page_count:
+            return
+        # set the start value
+        start = start or 0
+        while start < 0:
+            start += self.page_count
+        if start not in range(self.page_count):
+            raise ValueError("bad start page number")
+
+        # set the stop value
+        stop = stop if stop is not None and stop <= self.page_count else self.page_count
+
+        # set the step value
+        if step == 0:
+            raise ValueError("arg 3 must not be zero")
+        if step is None:
+            if start > stop:
+                step = -1
+            else:
+                step = 1
+
+        for pno in range(start, stop, step):
+            yield (self.load_page(pno))
+
+    def pdf_catalog(self):
+        """Get xref of PDF catalog."""
+        pdf = _as_pdf_document(self, required=0)
+        xref = 0
+        if not pdf.m_internal:
+            return xref
+        root = mupdf.pdf_dict_get(mupdf.pdf_trailer(pdf), PDF_NAME('Root'))
+        xref = mupdf.pdf_to_num(root)
+        return xref
+
+    def pdf_trailer(self, compressed=0, ascii=0):
+        """Get PDF trailer as a string."""
+        return self.xref_object(-1, compressed=compressed, ascii=ascii)
+
+    @property
+    def permissions(self):
+        """Document permissions."""
+        if self.is_encrypted:
+            return 0
+        doc =self.this
+        pdf = mupdf.pdf_document_from_fz_document(doc)
+
+        # for PDF return result of standard function
+        if pdf.m_internal:
+            return mupdf.pdf_document_permissions(pdf)
+
+        # otherwise simulate the PDF return value
+        perm = 0xFFFFFFFC   # all permissions granted
+        # now switch off where needed
+        if not mupdf.fz_has_permission(doc, mupdf.FZ_PERMISSION_PRINT):
+            perm = perm ^ mupdf.PDF_PERM_PRINT
+        if not mupdf.fz_has_permission(doc, mupdf.FZ_PERMISSION_EDIT):
+            perm = perm ^ mupdf.PDF_PERM_MODIFY
+        if not mupdf.fz_has_permission(doc, mupdf.FZ_PERMISSION_COPY):
+            perm = perm ^ mupdf.PDF_PERM_COPY
+        if not mupdf.fz_has_permission(doc, mupdf.FZ_PERMISSION_ANNOTATE):
+            perm = perm ^ mupdf.PDF_PERM_ANNOTATE
+        return perm
+
+    def prev_location(self, page_id):
+
+        """Get (chapter, page) of previous page."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        if type(page_id) is int:
+            page_id = (0, page_id)
+        if page_id not in self:
+            raise ValueError("page id not in document")
+        if page_id  == (0, 0):
+            return ()
+        chapter, pno = page_id
+        loc = mupdf.fz_make_location(chapter, pno)
+        prev_loc = mupdf.fz_previous_page(self.this, loc)
+        return prev_loc.chapter, prev_loc.page
+
+    def reload_page(self, page: Page) -> Page:
+        """Make a fresh copy of a page."""
+        old_annots = {}  # copy annot references to here
+        pno = page.number  # save the page number
+        for k, v in page._annot_refs.items():  # save the annot dictionary
+            old_annots[k] = v
+        
+        # When we call `self.load_page()` below, it will end up in
+        # fz_load_chapter_page(), which will return any matching page in the
+        # document's list of non-ref-counted loaded pages, instead of actually
+        # reloading the page.
+        #
+        # We want to assert that we have actually reloaded the fz_page, and not
+        # simply returned the same `fz_page*` pointer from the document's list
+        # of non-ref-counted loaded pages.
+        #
+        # So we first remove our reference to the `fz_page*`. This will
+        # decrement .refs, and if .refs was 1, this is guaranteed to free the
+        # `fz_page*` and remove it from the document's list if it was there. So
+        # we are guaranteed that our returned `fz_page*` is from a genuine
+        # reload, even if it happens to reuse the original block of memory.
+        #
+        # However if the original .refs is greater than one, there must be
+        # other references to the `fz_page` somewhere, and we require that
+        # these other references are not keeping the page in the document's
+        # list.  We check that we are returning a newly loaded page by
+        # asserting that our returned `fz_page*` is different from the original
+        # `fz_page*` - the original was not freed, so a new `fz_page` cannot
+        # reuse the same block of memory.
+        #
+        
+        refs_old = page.this.m_internal.refs
+        m_internal_old = page.this.m_internal_value()
+        
+        page.this = None
+        page._erase()  # remove the page
+        page = None
+        TOOLS.store_shrink(100)
+        page = self.load_page(pno)  # reload the page
+
+        # copy annot refs over to the new dictionary
+        #page_proxy = weakref.proxy(page)
+        for k, v in old_annots.items():
+            annot = old_annots[k]
+            #annot.parent = page_proxy  # refresh parent to new page
+            page._annot_refs[k] = annot
+        if refs_old == 1:
+            # We know that `page.this = None` will have decremented the ref
+            # count to zero so we are guaranteed that the new `fz_page` is a
+            # new page even if it happens to have reused the same block of
+            # memory.
+            pass
+        else:
+            # Check that the new `fz_page*` is different from the original.
+            m_internal_new = page.this.m_internal_value()
+            assert m_internal_new != m_internal_old, \
+                    f'{refs_old=} {m_internal_old=:#x} {m_internal_new=:#x}'
+        return page
+
+    def resolve_link(self, uri=None, chapters=0):
+        """Calculate internal link destination.
+
+        Args:
+            uri: (str) some Link.uri
+            chapters: (bool) whether to use (chapter, page) format
+        Returns:
+            (page_id, x, y) where x, y are point coordinates on the page.
+            page_id is either page number (if chapters=0), or (chapter, pno).
+        """
+        if not uri:
+            if chapters:
+                return (-1, -1), 0, 0
+            return -1, 0, 0
+        try:
+            loc, xp, yp = mupdf.fz_resolve_link(self.this, uri)
+        except Exception:
+            if g_exceptions_verbose:    exception_info()
+            if chapters:
+                return (-1, -1), 0, 0
+            return -1, 0, 0
+        if chapters:
+            return (loc.chapter, loc.page), xp, yp
+        pno = mupdf.fz_page_number_from_location(self.this, loc)
+        return pno, xp, yp
+
+    def rewrite_images(
+        self,
+        dpi_threshold=None,
+        dpi_target=0,
+        quality=0,
+        lossy=True,
+        lossless=True,
+        bitonal=True,
+        color=True,
+        gray=True,
+        set_to_gray=False,
+        options=None,
+    ):
+        """Rewrite images in a PDF document.
+
+        The typical use case is to reduce the size of the PDF by recompressing
+        images. Default parameters will convert all images to JPEG where
+        possible, using the specified resolutions and quality. Exclude
+        undesired images by setting parameters to False.
+        Args:
+            dpi_threshold: look at images with a larger DPI only.
+            dpi_target: change eligible images to this DPI.
+            quality: Quality of the recompressed images (0-100).
+            lossy: process lossy image types (e.g. JPEG).
+            lossless: process lossless image types (e.g. PNG).
+            bitonal: process black-and-white images (e.g. FAX)
+            color: process colored images.
+            gray: process gray images.
+            set_to_gray: whether to change the PDF to gray at process start.
+            options: (PdfImageRewriterOptions) Custom options for image
+                    rewriting (optional). Expert use only. If provided, other
+                    parameters are ignored, except set_to_gray.
+        """
+        quality_str = str(quality)
+        if not dpi_threshold:
+            dpi_threshold = dpi_target = 0
+        if dpi_target > 0 and dpi_target >= dpi_threshold:
+            raise ValueError("{dpi_target=} must be less than {dpi_threshold=}")
+        template_opts = mupdf.PdfImageRewriterOptions()
+        dir1 = set(dir(template_opts))  # for checking that only existing options are set
+        if not options:
+            opts = mupdf.PdfImageRewriterOptions()
+            if bitonal:
+                opts.bitonal_image_recompress_method = mupdf.FZ_RECOMPRESS_FAX
+                opts.bitonal_image_subsample_method = mupdf.FZ_SUBSAMPLE_AVERAGE
+                opts.bitonal_image_subsample_to = dpi_target
+                opts.bitonal_image_recompress_quality = quality_str
+                opts.bitonal_image_subsample_threshold = dpi_threshold
+            if color:
+                if lossless:
+                    opts.color_lossless_image_recompress_method = mupdf.FZ_RECOMPRESS_JPEG
+                    opts.color_lossless_image_subsample_method = mupdf.FZ_SUBSAMPLE_AVERAGE
+                    opts.color_lossless_image_subsample_to = dpi_target
+                    opts.color_lossless_image_subsample_threshold = dpi_threshold
+                    opts.color_lossless_image_recompress_quality = quality_str
+                if lossy:
+                    opts.color_lossy_image_recompress_method = mupdf.FZ_RECOMPRESS_JPEG
+                    opts.color_lossy_image_subsample_method = mupdf.FZ_SUBSAMPLE_AVERAGE
+                    opts.color_lossy_image_subsample_threshold = dpi_threshold
+                    opts.color_lossy_image_subsample_to = dpi_target
+                    opts.color_lossy_image_recompress_quality = quality_str
+            if gray:
+                if lossless:
+                    opts.gray_lossless_image_recompress_method = mupdf.FZ_RECOMPRESS_JPEG
+                    opts.gray_lossless_image_subsample_method = mupdf.FZ_SUBSAMPLE_AVERAGE
+                    opts.gray_lossless_image_subsample_to = dpi_target
+                    opts.gray_lossless_image_subsample_threshold = dpi_threshold
+                    opts.gray_lossless_image_recompress_quality = quality_str
+                if lossy:
+                    opts.gray_lossy_image_recompress_method = mupdf.FZ_RECOMPRESS_JPEG
+                    opts.gray_lossy_image_subsample_method = mupdf.FZ_SUBSAMPLE_AVERAGE
+                    opts.gray_lossy_image_subsample_threshold = dpi_threshold
+                    opts.gray_lossy_image_subsample_to = dpi_target
+                    opts.gray_lossy_image_recompress_quality = quality_str
+        else:
+            opts = options
+
+        dir2 = set(dir(opts))  # checking that only possible options were used
+        invalid_options = dir2 - dir1
+        if invalid_options:
+            raise ValueError(f"Invalid options: {invalid_options}")
+
+        if set_to_gray:
+            self.recolor(1)
+        pdf = _as_pdf_document(self)
+        mupdf.pdf_rewrite_images(pdf, opts)
+
+    def recolor(self, components=1):
+        """Change the color component count on all pages.
+
+        Args:
+            components: (int) desired color component count, one of 1, 3, 4.
+
+        Invokes the same-named method for all pages.
+        """
+        if not self.is_pdf:
+            raise ValueError("is no PDF")
+        for i in range(self.page_count):
+            self.load_page(i).recolor(components)
+
+    def resolve_names(self):
+        """Convert the PDF's destination names into a Python dict.
+
+        The only parameter is the pymupdf.Document.
+        All names found in the catalog under keys "/Dests" and "/Names/Dests" are
+        being included.
+
+        Returns:
+            A dcitionary with the following layout:
+            - key: (str) the name
+            - value: (dict) with the following layout:
+                * "page":  target page number (0-based). If no page number found -1.
+                * "to": (x, y) target point on page - currently in PDF coordinates,
+                        i.e. point (0,0) is the bottom-left of the page.
+                * "zoom": (float) the zoom factor
+                * "dest": (str) only occurs if the target location on the page has
+                        not been provided as "/XYZ" or if no page number was found.
+            Examples:
+            {'__bookmark_1': {'page': 0, 'to': (0.0, 541.0), 'zoom': 0.0},
+            '__bookmark_2': {'page': 0, 'to': (0.0, 481.45), 'zoom': 0.0}}
+
+            or
+
+            '21154a7c20684ceb91f9c9adc3b677c40': {'page': -1, 'dest': '/XYZ 15.75 1486 0'}, ...
+        """
+        if hasattr(self, "_resolved_names"):  # do not execute multiple times!
+            return self._resolved_names
+        # this is a backward listing of page xref to page number
+        page_xrefs = {self.page_xref(i): i for i in range(self.page_count)}
+
+        def obj_string(obj):
+            """Return string version of a PDF object definition."""
+            buffer = mupdf.fz_new_buffer(512)
+            output = mupdf.FzOutput(buffer)
+            mupdf.pdf_print_obj(output, obj, 1, 0)
+            output.fz_close_output()
+            return JM_UnicodeFromBuffer(buffer)
+
+        def get_array(val):
+            """Generate value of one item of the names dictionary."""
+            templ_dict = {"page": -1, "dest": ""}  # value template
+            if val.pdf_is_indirect():
+                val = mupdf.pdf_resolve_indirect(val)
+            if val.pdf_is_array():
+                array = obj_string(val)
+            elif val.pdf_is_dict():
+                array = obj_string(mupdf.pdf_dict_gets(val, "D"))
+            else:  # if all fails return the empty template
+                return templ_dict
+
+            # replace PDF "null" by zero, omit the square brackets
+            array = array.replace("null", "0")[1:-1]
+
+            # find stuff before first "/"
+            idx = array.find("/")
+            if idx < 1:  # this has no target page spec
+                templ_dict["dest"] = array  # return the orig. string
+                return templ_dict
+
+            subval = array[:idx].strip()  # stuff before "/"
+            array = array[idx:]  # stuff from "/" onwards
+            templ_dict["dest"] = array
+            # if we start with /XYZ: extract x, y, zoom
+            # 1, 2 or 3 of these values may actually be supplied
+            if array.startswith("/XYZ"):
+                del templ_dict["dest"]  # don't return orig string in this case
+
+                # make a list of the 3 tokens following "/XYZ"
+                array_list = array.split()[1:4]  # omit "/XYZ"
+
+                # fill up missing tokens with "0" strings
+                while len(array_list) < 3:  # fill up if too short
+                    array_list.append("0")  # add missing values
+
+                # make list of 3 floats: x, y and zoom
+                t = list(map(float, array_list))  # the resulting x, y, z values
+                templ_dict["to"] = (t[0], t[1])
+                templ_dict["zoom"] = t[2]
+
+            # extract page number
+            if subval.endswith("0 R"):  # page xref given?
+                templ_dict["page"] = page_xrefs.get(int(subval.split()[0]),-1)
+            else:  # naked page number given
+                templ_dict["page"] = int(subval)
+            return templ_dict
+
+        def fill_dict(dest_dict, pdf_dict):
+            """Generate name resolution items for pdf_dict.
+
+            This may be either "/Names/Dests" or just "/Dests"
+            """
+            # length of the PDF dictionary
+            name_count = mupdf.pdf_dict_len(pdf_dict)
+
+            # extract key-val of each dict item
+            for i in range(name_count):
+                key = mupdf.pdf_dict_get_key(pdf_dict, i)
+                val = mupdf.pdf_dict_get_val(pdf_dict, i)
+                if key.pdf_is_name():  # this should always be true!
+                    dict_key = key.pdf_to_name()
+                else:
+                    message(f"key {i} is no /Name")
+                    dict_key = None
+
+                if dict_key:
+                    dest_dict[dict_key] = get_array(val)  # store key/value in dict
+
+        # access underlying PDF document of fz Document
+        pdf = mupdf.pdf_document_from_fz_document(self)
+
+        # access PDF catalog
+        catalog = mupdf.pdf_dict_gets(mupdf.pdf_trailer(pdf), "Root")
+
+        dest_dict = {}
+
+        # make PDF_NAME(Dests)
+        dests = mupdf.pdf_new_name("Dests")
+
+        # extract destinations old style (PDF 1.1)
+        old_dests = mupdf.pdf_dict_get(catalog, dests)
+        if old_dests.pdf_is_dict():
+            fill_dict(dest_dict, old_dests)
+
+        # extract destinations new style (PDF 1.2+)
+        tree = mupdf.pdf_load_name_tree(pdf, dests)
+        if tree.pdf_is_dict():
+            fill_dict(dest_dict, tree)
+
+        self._resolved_names = dest_dict  # store result or reuse
+        return dest_dict
+
+    def save(
+            self,
+            filename,
+            garbage=0,
+            clean=0,
+            deflate=0,
+            deflate_images=0,
+            deflate_fonts=0,
+            incremental=0,
+            ascii=0,
+            expand=0,
+            linear=0,
+            no_new_id=0,
+            appearance=0,
+            pretty=0,
+            encryption=1,
+            permissions=4095,
+            owner_pw=None,
+            user_pw=None,
+            preserve_metadata=1,
+            use_objstms=0,
+            compression_effort=0,
+            ):
+        # From %pythonprepend save
+        #
+        """Save PDF to file, pathlib.Path or file pointer."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        if type(filename) is str:
+            pass
+        elif hasattr(filename, "open"):  # assume: pathlib.Path
+            filename = str(filename)
+        elif hasattr(filename, "name"):  # assume: file object
+            filename = filename.name
+        elif not hasattr(filename, "seek"):  # assume file object
+            raise ValueError("filename must be str, Path or file object")
+        if filename == self.name and not incremental:
+            raise ValueError("save to original must be incremental")
+        if linear and use_objstms:
+            raise ValueError("'linear' and 'use_objstms' cannot both be requested")
+        if self.page_count < 1:
+            raise ValueError("cannot save with zero pages")
+        if incremental:
+            if self.name != filename or self.stream:
+                raise ValueError("incremental needs original file")
+        if user_pw and len(user_pw) > 40 or owner_pw and len(owner_pw) > 40:
+            raise ValueError("password length must not exceed 40")
+        
+        pdf = _as_pdf_document(self)
+        opts = mupdf.PdfWriteOptions()
+        opts.do_incremental = incremental
+        opts.do_ascii = ascii
+        opts.do_compress = deflate
+        opts.do_compress_images = deflate_images
+        opts.do_compress_fonts = deflate_fonts
+        opts.do_decompress = expand
+        opts.do_garbage = garbage
+        opts.do_pretty = pretty
+        opts.do_linear = linear
+        opts.do_clean = clean
+        opts.do_sanitize = clean
+        opts.dont_regenerate_id = no_new_id
+        opts.do_appearance = appearance
+        opts.do_encrypt = encryption
+        opts.permissions = permissions
+        if owner_pw is not None:
+            opts.opwd_utf8_set_value(owner_pw)
+        elif user_pw is not None:
+            opts.opwd_utf8_set_value(user_pw)
+        if user_pw is not None:
+            opts.upwd_utf8_set_value(user_pw)
+        opts.do_preserve_metadata = preserve_metadata
+        opts.do_use_objstms = use_objstms
+        opts.compression_effort = compression_effort
+
+        out = None
+        pdf.m_internal.resynth_required = 0
+        JM_embedded_clean(pdf)
+        if no_new_id == 0:
+            JM_ensure_identity(pdf)
+        if isinstance(filename, str):
+            #log( 'calling mupdf.pdf_save_document()')
+            mupdf.pdf_save_document(pdf, filename, opts)
+        else:
+            out = JM_new_output_fileptr(filename)
+            #log( f'{type(out)=} {type(out.this)=}')
+            mupdf.pdf_write_document(pdf, out, opts)
+            out.fz_close_output()
+
+    def save_snapshot(self, filename):
+        """Save a file snapshot suitable for journalling."""
+        if self.is_closed:
+            raise ValueError("doc is closed")
+        if type(filename) is str:
+            pass
+        elif hasattr(filename, "open"):  # assume: pathlib.Path
+            filename = str(filename)
+        elif hasattr(filename, "name"):  # assume: file object
+            filename = filename.name
+        else:
+            raise ValueError("filename must be str, Path or file object")
+        if filename == self.name:
+            raise ValueError("cannot snapshot to original")
+        pdf = _as_pdf_document(self)
+        mupdf.pdf_save_snapshot(pdf, filename)
+
+    def saveIncr(self):
+        """ Save PDF incrementally"""
+        return self.save(self.name, incremental=True, encryption=mupdf.PDF_ENCRYPT_KEEP)
+
+    def select(self, pyliste):
+        """Build sub-pdf with page numbers in the list."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        if not self.is_pdf:
+            raise ValueError("is no PDF")
+        if not hasattr(pyliste, "__getitem__"):
+            raise ValueError("sequence required")
+
+        valid_range = range(len(self))
+        if (len(pyliste) == 0
+            or min(pyliste) not in valid_range
+            or max(pyliste) not in valid_range
+        ):
+            raise ValueError("bad page number(s)")
+
+        # get underlying pdf document,
+        pdf = _as_pdf_document(self)
+        # create page sub-pdf via pdf_rearrange_pages2().
+        #
+        if mupdf_version_tuple >= (1, 25, 3):
+            # We use PDF_CLEAN_STRUCTURE_KEEP otherwise we lose structure tree
+            # which, for example, breaks test_3705.
+            mupdf.pdf_rearrange_pages2(pdf, pyliste, mupdf.PDF_CLEAN_STRUCTURE_KEEP)
+        else:
+            mupdf.pdf_rearrange_pages2(pdf, pyliste)
+
+        # remove any existing pages with their kids
+        self._reset_page_refs()
+
+    def set_language(self, language=None):
+        pdf = _as_pdf_document(self)
+        if not language:
+            lang = mupdf.FZ_LANG_UNSET
+        else:
+            lang = mupdf.fz_text_language_from_string(language)
+        mupdf.pdf_set_document_language(pdf, lang)
+        return True
+
+    def set_layer(self, config, basestate=None, on=None, off=None, rbgroups=None, locked=None):
+        """Set the PDF keys /ON, /OFF, /RBGroups of an OC layer."""
+        if self.is_closed:
+            raise ValueError("document closed")
+        ocgs = set(self.get_ocgs().keys())
+        if ocgs == set():
+            raise ValueError("document has no optional content")
+
+        if on:
+            if type(on) not in (list, tuple):
+                raise ValueError("bad type: 'on'")
+            s = set(on).difference(ocgs)
+            if s != set():
+                raise ValueError("bad OCGs in 'on': %s" % s)
+
+        if off:
+            if type(off) not in (list, tuple):
+                raise ValueError("bad type: 'off'")
+            s = set(off).difference(ocgs)
+            if s != set():
+                raise ValueError("bad OCGs in 'off': %s" % s)
+
+        if locked:
+            if type(locked) not in (list, tuple):
+                raise ValueError("bad type: 'locked'")
+            s = set(locked).difference(ocgs)
+            if s != set():
+                raise ValueError("bad OCGs in 'locked': %s" % s)
+
+        if rbgroups:
+            if type(rbgroups) not in (list, tuple):
+                raise ValueError("bad type: 'rbgroups'")
+            for x in rbgroups:
+                if not type(x) in (list, tuple):
+                    raise ValueError("bad RBGroup '%s'" % x)
+                s = set(x).difference(ocgs)
+                if s != set():
+                    raise ValueError("bad OCGs in RBGroup: %s" % s)
+
+        if basestate:
+            basestate = str(basestate).upper()
+            if basestate == "UNCHANGED":
+                basestate = "Unchanged"
+            if basestate not in ("ON", "OFF", "Unchanged"):
+                raise ValueError("bad 'basestate'")
+        pdf = _as_pdf_document(self)
+        ocp = mupdf.pdf_dict_getl(
+                mupdf.pdf_trailer( pdf),
+                PDF_NAME('Root'),
+                PDF_NAME('OCProperties'),
+                )
+        if not ocp.m_internal:
+            return
+        if config == -1:
+            obj = mupdf.pdf_dict_get( ocp, PDF_NAME('D'))
+        else:
+            obj = mupdf.pdf_array_get(
+                    mupdf.pdf_dict_get( ocp, PDF_NAME('Configs')),
+                    config,
+                    )
+        if not obj.m_internal:
+            raise ValueError( MSG_BAD_OC_CONFIG)
+        JM_set_ocg_arrays( obj, basestate, on, off, rbgroups, locked)
+        mupdf.ll_pdf_read_ocg( pdf.m_internal)
+
+    def set_layer_ui_config(self, number, action=0):
+        """Set / unset OC intent configuration."""
+        # The user might have given the name instead of sequence number,
+        # so select by that name and continue with corresp. number
+        if isinstance(number, str):
+            select = [ui["number"] for ui in self.layer_ui_configs() if ui["text"] == number]
+            if select == []:
+                raise ValueError(f"bad OCG '{number}'.")
+            number = select[0]  # this is the number for the name
+        pdf = _as_pdf_document(self)
+        if action == 1:
+            mupdf.pdf_toggle_layer_config_ui(pdf, number)
+        elif action == 2:
+            mupdf.pdf_deselect_layer_config_ui(pdf, number)
+        else:
+            mupdf.pdf_select_layer_config_ui(pdf, number)
+
+    def set_markinfo(self, markinfo: dict) -> bool:
+        """Set the PDF MarkInfo values."""
+        xref = self.pdf_catalog()
+        if xref == 0:
+            raise ValueError("not a PDF")
+        if not markinfo or not isinstance(markinfo, dict):
+            return False
+        valid = {"Marked": False, "UserProperties": False, "Suspects": False}
+
+        if not set(valid.keys()).issuperset(markinfo.keys()):
+            badkeys = f"bad MarkInfo key(s): {set(markinfo.keys()).difference(valid.keys())}"
+            raise ValueError(badkeys)
+        pdfdict = "<<"
+        valid.update(markinfo)
+        for key, value in valid.items():
+            value=str(value).lower()
+            if value not in ("true", "false"):
+                raise ValueError(f"bad key value '{key}': '{value}'")
+            pdfdict += f"/{key} {value}"
+        pdfdict += ">>"
+        self.xref_set_key(xref, "MarkInfo", pdfdict)
+        return True
+
+    def set_pagelayout(self, pagelayout: str):
+        """Set the PDF PageLayout value."""
+        valid = ("SinglePage", "OneColumn", "TwoColumnLeft", "TwoColumnRight", "TwoPageLeft", "TwoPageRight")
+        xref = self.pdf_catalog()
+        if xref == 0:
+            raise ValueError("not a PDF")
+        if not pagelayout:
+            raise ValueError("bad PageLayout value")
+        if pagelayout[0] == "/":
+            pagelayout = pagelayout[1:]
+        for v in valid:
+            if pagelayout.lower() == v.lower():
+                self.xref_set_key(xref, "PageLayout", f"/{v}")
+                return True
+        raise ValueError("bad PageLayout value")
+
+    def set_pagemode(self, pagemode: str):
+        """Set the PDF PageMode value."""
+        valid = ("UseNone", "UseOutlines", "UseThumbs", "FullScreen", "UseOC", "UseAttachments")
+        xref = self.pdf_catalog()
+        if xref == 0:
+            raise ValueError("not a PDF")
+        if not pagemode:
+            raise ValueError("bad PageMode value")
+        if pagemode[0] == "/":
+            pagemode = pagemode[1:]
+        for v in valid:
+            if pagemode.lower() == v.lower():
+                self.xref_set_key(xref, "PageMode", f"/{v}")
+                return True
+        raise ValueError("bad PageMode value")
+
+    def set_xml_metadata(self, metadata):
+        """Store XML document level metadata."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        root = mupdf.pdf_dict_get( mupdf.pdf_trailer( pdf), PDF_NAME('Root'))
+        if not root.m_internal:
+            RAISEPY( MSG_BAD_PDFROOT, JM_Exc_FileDataError)
+        res = mupdf.fz_new_buffer_from_copied_data( metadata.encode('utf-8'))
+        xml = mupdf.pdf_dict_get( root, PDF_NAME('Metadata'))
+        if xml.m_internal:
+            JM_update_stream( pdf, xml, res, 0)
+        else:
+            xml = mupdf.pdf_add_stream( pdf, res, mupdf.PdfObj(), 0)
+            mupdf.pdf_dict_put( xml, PDF_NAME('Type'), PDF_NAME('Metadata'))
+            mupdf.pdf_dict_put( xml, PDF_NAME('Subtype'), PDF_NAME('XML'))
+            mupdf.pdf_dict_put( root, PDF_NAME('Metadata'), xml)
+
+    def switch_layer(self, config, as_default=0):
+        """Activate an OC layer."""
+        pdf = _as_pdf_document(self)
+        cfgs = mupdf.pdf_dict_getl(
+                mupdf.pdf_trailer( pdf),
+                PDF_NAME('Root'),
+                PDF_NAME('OCProperties'),
+                PDF_NAME('Configs')
+                )
+        if not mupdf.pdf_is_array( cfgs) or not mupdf.pdf_array_len( cfgs):
+            if config < 1:
+                return
+            raise ValueError( MSG_BAD_OC_LAYER)
+        if config < 0:
+            return
+        mupdf.pdf_select_layer_config( pdf, config)
+        if as_default:
+            mupdf.pdf_set_layer_config_as_default( pdf)
+            mupdf.ll_pdf_read_ocg( pdf.m_internal)
+
+    def update_object(self, xref, text, page=None):
+        """Replace object definition source."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        xreflen = mupdf.pdf_xref_len(pdf)
+        if not _INRANGE(xref, 1, xreflen-1):
+            RAISEPY("bad xref", MSG_BAD_XREF)
+        ENSURE_OPERATION(pdf)
+        # create new object with passed-in string
+        new_obj = JM_pdf_obj_from_str(pdf, text)
+        mupdf.pdf_update_object(pdf, xref, new_obj)
+        if page:
+            JM_refresh_links( _as_pdf_page(page))
+
+    def update_stream(self, xref=0, stream=None, new=1, compress=1):
+        """Replace xref stream part."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        xreflen = mupdf.pdf_xref_len(pdf)
+        if xref < 1 or xref > xreflen:
+            raise ValueError( MSG_BAD_XREF)
+        # get the object
+        obj = mupdf.pdf_new_indirect(pdf, xref, 0)
+        if not mupdf.pdf_is_dict(obj):
+            raise ValueError( MSG_IS_NO_DICT)
+        res = JM_BufferFromBytes(stream)
+        if not res.m_internal:
+            raise TypeError( MSG_BAD_BUFFER)
+        JM_update_stream(pdf, obj, res, compress)
+        pdf.dirty = 1
+
+    @property
+    def version_count(self):
+        '''
+        Count versions of PDF document.
+        '''
+        pdf = _as_pdf_document(self, required=0)
+        if pdf.m_internal:
+            return mupdf.pdf_count_versions(pdf)
+        return 0
+
+    def write(
+            self,
+            garbage=False,
+            clean=False,
+            deflate=False,
+            deflate_images=False,
+            deflate_fonts=False,
+            incremental=False,
+            ascii=False,
+            expand=False,
+            linear=False,
+            no_new_id=False,
+            appearance=False,
+            pretty=False,
+            encryption=1,
+            permissions=4095,
+            owner_pw=None,
+            user_pw=None,
+            preserve_metadata=1,
+            use_objstms=0,
+            compression_effort=0,
+    ):
+        from io import BytesIO
+        bio = BytesIO()
+        self.save(
+                bio,
+                garbage=garbage,
+                clean=clean,
+                no_new_id=no_new_id,
+                appearance=appearance,
+                deflate=deflate,
+                deflate_images=deflate_images,
+                deflate_fonts=deflate_fonts,
+                incremental=incremental,
+                ascii=ascii,
+                expand=expand,
+                linear=linear,
+                pretty=pretty,
+                encryption=encryption,
+                permissions=permissions,
+                owner_pw=owner_pw,
+                user_pw=user_pw,
+                preserve_metadata=preserve_metadata,
+                use_objstms=use_objstms,
+                compression_effort=compression_effort,
+        )
+        return bio.getvalue()
+
+    @property
+    def xref(self):
+        """PDF xref number of page."""
+        CheckParent(self)
+        return self.parent.page_xref(self.number)
+
+    def xref_get_key(self, xref, key):
+        """Get PDF dict key value of object at 'xref'."""
+        pdf = _as_pdf_document(self)
+        xreflen = mupdf.pdf_xref_len(pdf)
+        if not _INRANGE(xref, 1, xreflen-1) and xref != -1:
+            raise ValueError( MSG_BAD_XREF)
+        if xref > 0:
+            obj = mupdf.pdf_load_object(pdf, xref)
+        else:
+            obj = mupdf.pdf_trailer(pdf)
+        if not obj.m_internal:
+            return ("null", "null")
+        subobj = mupdf.pdf_dict_getp(obj, key)
+        if not subobj.m_internal:
+            return ("null", "null")
+        text = None
+        if mupdf.pdf_is_indirect(subobj):
+            type = "xref"
+            text = "%i 0 R" % mupdf.pdf_to_num(subobj)
+        elif mupdf.pdf_is_array(subobj):
+            type = "array"
+        elif mupdf.pdf_is_dict(subobj):
+            type = "dict"
+        elif mupdf.pdf_is_int(subobj):
+            type = "int"
+            text = "%i" % mupdf.pdf_to_int(subobj)
+        elif mupdf.pdf_is_real(subobj):
+            type = "float"
+        elif mupdf.pdf_is_null(subobj):
+            type = "null"
+            text = "null"
+        elif mupdf.pdf_is_bool(subobj):
+            type = "bool"
+            if mupdf.pdf_to_bool(subobj):
+                text = "true"
+            else:
+                text = "false"
+        elif mupdf.pdf_is_name(subobj):
+            type = "name"
+            text = "/%s" % mupdf.pdf_to_name(subobj)
+        elif mupdf.pdf_is_string(subobj):
+            type = "string"
+            text = JM_UnicodeFromStr(mupdf.pdf_to_text_string(subobj))
+        else:
+            type = "unknown"
+        if text is None:
+            res = JM_object_to_buffer(subobj, 1, 0)
+            text = JM_UnicodeFromBuffer(res)
+        return (type, text)
+
+    def xref_get_keys(self, xref):
+        """Get the keys of PDF dict object at 'xref'. Use -1 for the PDF trailer."""
+        pdf = _as_pdf_document(self)
+        xreflen = mupdf.pdf_xref_len( pdf)
+        if not _INRANGE(xref, 1, xreflen-1) and xref != -1:
+            raise ValueError( MSG_BAD_XREF)
+        if xref > 0:
+            obj = mupdf.pdf_load_object( pdf, xref)
+        else:
+            obj = mupdf.pdf_trailer( pdf)
+        n = mupdf.pdf_dict_len( obj)
+        rc = []
+        if n == 0:
+            return rc
+        for i in range(n):
+            key = mupdf.pdf_to_name( mupdf.pdf_dict_get_key( obj, i))
+            rc.append(key)
+        return rc
+
+    def xref_is_font(self, xref):
+        """Check if xref is a font object."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        if self.xref_get_key(xref, "Type")[1] == "/Font":
+            return True
+        return False
+
+    def xref_is_image(self, xref):
+        """Check if xref is an image object."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        if self.xref_get_key(xref, "Subtype")[1] == "/Image":
+            return True
+        return False
+
+    def xref_is_stream(self, xref=0):
+        """Check if xref is a stream object."""
+        pdf = _as_pdf_document(self, required=0)
+        if not pdf.m_internal:
+            return False    # not a PDF
+        return bool(mupdf.pdf_obj_num_is_stream(pdf, xref))
+
+    def xref_is_xobject(self, xref):
+        """Check if xref is a form xobject."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        if self.xref_get_key(xref, "Subtype")[1] == "/Form":
+            return True
+        return False
+
+    def xref_length(self):
+        """Get length of xref table."""
+        xreflen = 0
+        pdf = _as_pdf_document(self, required=0)
+        if pdf.m_internal:
+            xreflen = mupdf.pdf_xref_len(pdf)
+        return xreflen
+
+    def xref_object(self, xref, compressed=0, ascii=0):
+        """Get xref object source as a string."""
+        if self.is_closed:
+            raise ValueError("document closed")
+        if g_use_extra:
+            ret = extra.xref_object( self.this, xref, compressed, ascii)
+            return ret
+        pdf = _as_pdf_document(self)
+        xreflen = mupdf.pdf_xref_len(pdf)
+        if not _INRANGE(xref, 1, xreflen-1) and xref != -1:
+            raise ValueError( MSG_BAD_XREF)
+        if xref > 0:
+            obj = mupdf.pdf_load_object(pdf, xref)
+        else:
+            obj = mupdf.pdf_trailer(pdf)
+        res = JM_object_to_buffer(mupdf.pdf_resolve_indirect(obj), compressed, ascii)
+        text = JM_EscapeStrFromBuffer(res)
+        return text
+
+    def xref_set_key(self, xref, key, value):
+        """Set the value of a PDF dictionary key."""
+        if self.is_closed:
+            raise ValueError("document closed")
+
+        if not key or not isinstance(key, str) or INVALID_NAME_CHARS.intersection(key) not in (set(), {"/"}):
+            raise ValueError("bad 'key'")
+        if not isinstance(value, str) or not value or value[0] == "/" and INVALID_NAME_CHARS.intersection(value[1:]) != set():
+            raise ValueError("bad 'value'")
+
+        pdf = _as_pdf_document(self)
+        xreflen = mupdf.pdf_xref_len(pdf)
+        #if not _INRANGE(xref, 1, xreflen-1) and xref != -1:
+        #    THROWMSG("bad xref")
+        #if len(value) == 0:
+        #    THROWMSG("bad 'value'")
+        #if len(key) == 0:
+        #    THROWMSG("bad 'key'")
+        if not _INRANGE(xref, 1, xreflen-1) and xref != -1:
+            raise ValueError( MSG_BAD_XREF)
+        if xref != -1:
+            obj = mupdf.pdf_load_object(pdf, xref)
+        else:
+            obj = mupdf.pdf_trailer(pdf)
+        new_obj = JM_set_object_value(obj, key, value)
+        if not new_obj.m_internal:
+            return  # did not work: skip update
+        if xref != -1:
+            mupdf.pdf_update_object(pdf, xref, new_obj)
+        else:
+            n = mupdf.pdf_dict_len(new_obj)
+            for i in range(n):
+                mupdf.pdf_dict_put(
+                        obj,
+                        mupdf.pdf_dict_get_key(new_obj, i),
+                        mupdf.pdf_dict_get_val(new_obj, i),
+                        )
+
+    def xref_stream(self, xref):
+        """Get decompressed xref stream."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        xreflen = mupdf.pdf_xref_len( pdf)
+        if not _INRANGE(xref, 1, xreflen-1) and xref != -1:
+            raise ValueError( MSG_BAD_XREF)
+        if xref >= 0:
+            obj = mupdf.pdf_new_indirect( pdf, xref, 0)
+        else:
+            obj = mupdf.pdf_trailer( pdf)
+        r = None
+        if mupdf.pdf_is_stream( obj):
+            res = mupdf.pdf_load_stream_number( pdf, xref)
+            r = JM_BinFromBuffer( res)
+        return r
+
+    def xref_stream_raw(self, xref):
+        """Get xref stream without decompression."""
+        if self.is_closed or self.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        pdf = _as_pdf_document(self)
+        xreflen = mupdf.pdf_xref_len( pdf)
+        if not _INRANGE(xref, 1, xreflen-1) and xref != -1:
+            raise ValueError( MSG_BAD_XREF)
+        if xref >= 0:
+            obj = mupdf.pdf_new_indirect( pdf, xref, 0)
+        else:
+            obj = mupdf.pdf_trailer( pdf)
+        r = None
+        if mupdf.pdf_is_stream( obj):
+            res = mupdf.pdf_load_raw_stream_number( pdf, xref)
+            r = JM_BinFromBuffer( res)
+        return r
+
+    def xref_xml_metadata(self):
+        """Get xref of document XML metadata."""
+        pdf = _as_pdf_document(self)
+        root = mupdf.pdf_dict_get( mupdf.pdf_trailer( pdf), PDF_NAME('Root'))
+        if not root.m_internal:
+            RAISEPY( MSG_BAD_PDFROOT, JM_Exc_FileDataError)
+        xml = mupdf.pdf_dict_get( root, PDF_NAME('Metadata'))
+        xref = 0
+        if xml.m_internal:
+            xref = mupdf.pdf_to_num( xml)
+        return xref
+    
+    __slots__ = ('this', 'page_count2', 'this_is_pdf', '__dict__')
+    
+    outline = property(lambda self: self._outline)
+    tobytes = write
+    is_stream = xref_is_stream
+
+open = Document
+
+
+class DocumentWriter:
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+    def __init__(self, path, options=''):
+        if isinstance( path, str):
+            pass
+        elif hasattr( path, 'absolute'):
+            path = str( path)
+        elif hasattr( path, 'name'):
+            path = path.name
+        if isinstance( path, str):
+            self.this = mupdf.FzDocumentWriter( path, options, mupdf.FzDocumentWriter.PathType_PDF)
+        else:
+            # Need to keep the Python JM_new_output_fileptr_Output instance
+            # alive for the lifetime of this DocumentWriter, otherwise calls
+            # to virtual methods implemented in Python fail. So we make it a
+            # member of this DocumentWriter.
+            #
+            # Unrelated to this, mupdf.FzDocumentWriter will set
+            # self._out.m_internal to null because ownership is passed in.
+            #
+            out = JM_new_output_fileptr( path)
+            self.this = mupdf.FzDocumentWriter( out, options, mupdf.FzDocumentWriter.OutputType_PDF)
+            assert out.m_internal_value() == 0
+            assert hasattr( self.this, '_out')
+    
+    def begin_page( self, mediabox):
+        mediabox2 = JM_rect_from_py(mediabox)
+        device = mupdf.fz_begin_page( self.this, mediabox2)
+        device_wrapper = DeviceWrapper( device)
+        return device_wrapper
+    
+    def close( self):
+        mupdf.fz_close_document_writer( self.this)
+        
+    def end_page( self):
+        mupdf.fz_end_page( self.this)
+
+
+class Font:
+
+    def __del__(self):
+        if type(self) is not Font:
+            return None
+
+    def __init__(
+            self,
+            fontname=None,
+            fontfile=None,
+            fontbuffer=None,
+            script=0,
+            language=None,
+            ordering=-1,
+            is_bold=0,
+            is_italic=0,
+            is_serif=0,
+            embed=1,
+            ):
+        
+        if fontbuffer:
+            if hasattr(fontbuffer, "getvalue"):
+                fontbuffer = fontbuffer.getvalue()
+            elif isinstance(fontbuffer, bytearray):
+                fontbuffer = bytes(fontbuffer)
+            if not isinstance(fontbuffer, bytes):
+                raise ValueError("bad type: 'fontbuffer'")
+        
+        if isinstance(fontname, str):
+            fname_lower = fontname.lower()
+            if "/" in fname_lower or "\\" in fname_lower or "." in fname_lower:
+                message("Warning: did you mean a fontfile?")
+
+            if fname_lower in ("cjk", "china-t", "china-ts"):
+                ordering = 0
+
+            elif fname_lower.startswith("china-s"):
+                ordering = 1
+            elif fname_lower.startswith("korea"):
+                ordering = 3
+            elif fname_lower.startswith("japan"):
+                ordering = 2
+            elif fname_lower in fitz_fontdescriptors.keys():
+                import pymupdf_fonts  # optional fonts
+                fontbuffer = pymupdf_fonts.myfont(fname_lower)  # make a copy
+                fontname = None  # ensure using fontbuffer only
+                del pymupdf_fonts  # remove package again
+
+            elif ordering < 0:
+                fontname = Base14_fontdict.get(fontname, fontname)
+
+        lang = mupdf.fz_text_language_from_string(language)
+        font = JM_get_font(fontname, fontfile,
+                   fontbuffer, script, lang, ordering,
+                   is_bold, is_italic, is_serif, embed)
+        self.this = font
+
+    def __repr__(self):
+        return "Font('%s')" % self.name
+
+    @property
+    def ascender(self):
+        """Return the glyph ascender value."""
+        return mupdf.fz_font_ascender(self.this)
+
+    @property
+    def bbox(self):
+        return self.this.fz_font_bbox()
+    
+    @property
+    def buffer(self):
+        buffer_ = mupdf.FzBuffer( mupdf.ll_fz_keep_buffer( self.this.m_internal.buffer))
+        return mupdf.fz_buffer_extract_copy( buffer_)
+
+    def char_lengths(self, text, fontsize=11, language=None, script=0, wmode=0, small_caps=0):
+        """Return tuple of char lengths of unicode 'text' under a fontsize."""
+        lang = mupdf.fz_text_language_from_string(language)
+        rc = []
+        for ch in text:
+            c = ord(ch)
+            if small_caps:
+                gid = mupdf.fz_encode_character_sc(self.this, c)
+                if gid >= 0:
+                    font = self.this
+            else:
+                gid, font = mupdf.fz_encode_character_with_fallback(self.this, c, script, lang)
+            rc.append(fontsize * mupdf.fz_advance_glyph(font, gid, wmode))
+        return rc
+
+    @property
+    def descender(self):
+        """Return the glyph descender value."""
+        return mupdf.fz_font_descender(self.this)
+
+    @property
+    def flags(self):
+        f = mupdf.ll_fz_font_flags(self.this.m_internal)
+        if not f:
+            return
+        assert isinstance( f, mupdf.fz_font_flags_t)
+        #log( '{=f}')
+        if mupdf_cppyy:
+            # cppyy includes remaining higher bits.
+            v = [f.is_mono]
+            def b(bits):
+                ret = v[0] & ((1 << bits)-1)
+                v[0] = v[0] >> bits
+                return ret
+            is_mono = b(1)
+            is_serif = b(1)
+            is_bold = b(1)
+            is_italic = b(1)
+            ft_substitute = b(1)
+            ft_stretch = b(1)
+            fake_bold = b(1)
+            fake_italic = b(1)
+            has_opentype = b(1)
+            invalid_bbox = b(1)
+            cjk_lang = b(1)
+            embed = b(1)
+            never_embed = b(1)
+        return {
+                "mono":         is_mono if mupdf_cppyy else f.is_mono,
+                "serif":        is_serif if mupdf_cppyy else f.is_serif,
+                "bold":         is_bold if mupdf_cppyy else f.is_bold,
+                "italic":       is_italic if mupdf_cppyy else f.is_italic,
+                "substitute":   ft_substitute if mupdf_cppyy else f.ft_substitute,
+                "stretch":      ft_stretch if mupdf_cppyy else f.ft_stretch,
+                "fake-bold":    fake_bold if mupdf_cppyy else f.fake_bold,
+                "fake-italic":  fake_italic if mupdf_cppyy else f.fake_italic,
+                "opentype":     has_opentype if mupdf_cppyy else f.has_opentype,
+                "invalid-bbox": invalid_bbox if mupdf_cppyy else f.invalid_bbox,
+                'cjk':          cjk_lang if mupdf_cppyy else f.cjk,
+                'cjk-lang':     cjk_lang if mupdf_cppyy else f.cjk_lang,
+                'embed':        embed if mupdf_cppyy else f.embed,
+                'never-embed':  never_embed if mupdf_cppyy else f.never_embed,
+                }
+
+    def glyph_advance(self, chr_, language=None, script=0, wmode=0, small_caps=0):
+        """Return the glyph width of a unicode (font size 1)."""
+        lang = mupdf.fz_text_language_from_string(language)
+        if small_caps:
+            gid = mupdf.fz_encode_character_sc(self.this, chr_)
+            if gid >= 0:
+                font = self.this
+        else:
+            gid, font = mupdf.fz_encode_character_with_fallback(self.this, chr_, script, lang)
+        return mupdf.fz_advance_glyph(font, gid, wmode)
+
+    def glyph_bbox(self, char, language=None, script=0, small_caps=0):
+        """Return the glyph bbox of a unicode (font size 1)."""
+        lang = mupdf.fz_text_language_from_string(language)
+        if small_caps:
+            gid = mupdf.fz_encode_character_sc( self.this, char)
+            if gid >= 0:
+                font = self.this
+        else:
+            gid, font = mupdf.fz_encode_character_with_fallback( self.this, char, script, lang)
+        return Rect(mupdf.fz_bound_glyph( font, gid, mupdf.FzMatrix()))
+
+    @property
+    def glyph_count(self):
+        return self.this.m_internal.glyph_count
+
+    def glyph_name_to_unicode(self, name):
+        """Return the unicode for a glyph name."""
+        return glyph_name_to_unicode(name)
+
+    def has_glyph(self, chr, language=None, script=0, fallback=0, small_caps=0):
+        """Check whether font has a glyph for this unicode."""
+        if fallback:
+            lang = mupdf.fz_text_language_from_string(language)
+            gid, font = mupdf.fz_encode_character_with_fallback(self.this, chr, script, lang)
+        else:
+            if small_caps:
+                gid = mupdf.fz_encode_character_sc(self.this, chr)
+            else:
+                gid = mupdf.fz_encode_character(self.this, chr)
+        return gid
+
+    @property
+    def is_bold(self):
+        return mupdf.fz_font_is_bold( self.this)
+
+    @property
+    def is_italic(self):
+        return mupdf.fz_font_is_italic( self.this)
+
+    @property
+    def is_monospaced(self):
+        return mupdf.fz_font_is_monospaced( self.this)
+
+    @property
+    def is_serif(self):
+        return mupdf.fz_font_is_serif( self.this)
+
+    @property
+    def is_writable(self):
+        return True # see pymupdf commit ef4056ee4da2
+        font = self.this
+        flags = mupdf.ll_fz_font_flags(font.m_internal)
+        if mupdf_cppyy:
+            # cppyy doesn't handle bitfields correctly.
+            import cppyy
+            ft_substitute = cppyy.gbl.mupdf_mfz_font_flags_ft_substitute( flags)
+        else:
+            ft_substitute = flags.ft_substitute
+        
+        if ( mupdf.ll_fz_font_t3_procs(font.m_internal)
+                or ft_substitute
+                or not mupdf.pdf_font_writing_supported(font)
+                ):
+            return False
+        return True
+
+    @property
+    def name(self):
+        ret = mupdf.fz_font_name(self.this)
+        #log( '{ret=}')
+        return ret
+
+    def text_length(self, text, fontsize=11, language=None, script=0, wmode=0, small_caps=0):
+        """Return length of unicode 'text' under a fontsize."""
+        thisfont = self.this
+        lang = mupdf.fz_text_language_from_string(language)
+        rc = 0
+        if not isinstance(text, str):
+            raise TypeError( MSG_BAD_TEXT)
+        for ch in text:
+            c = ord(ch)
+            if small_caps:
+                gid = mupdf.fz_encode_character_sc(thisfont, c)
+                if gid >= 0:
+                    font = thisfont
+            else:
+                gid, font = mupdf.fz_encode_character_with_fallback(thisfont, c, script, lang)
+            rc += mupdf.fz_advance_glyph(font, gid, wmode)
+        rc *= fontsize
+        return rc
+
+    def unicode_to_glyph_name(self, ch):
+        """Return the glyph name for a unicode."""
+        return unicode_to_glyph_name(ch)
+
+    def valid_codepoints(self):
+        '''
+        Returns sorted list of valid unicodes of a fz_font.
+        '''
+        ucs_gids = mupdf.fz_enumerate_font_cmap2(self.this)
+        ucss = [i.ucs for i in ucs_gids]
+        ucss_unique = set(ucss)
+        ucss_unique_sorted = sorted(ucss_unique)
+        return ucss_unique_sorted
+
+
+class Graftmap:
+
+    def __del__(self):
+        if not type(self) is Graftmap:
+            return
+        self.thisown = False
+
+    def __init__(self, doc):
+        dst = _as_pdf_document(doc)
+        map_ = mupdf.pdf_new_graft_map(dst)
+        self.this = map_
+        self.thisown = True
+
+
+class Link:
+    def __del__(self):
+        self._erase()
+
+    def __init__( self, this):
+        assert isinstance( this, mupdf.FzLink)
+        self.this = this
+
+    def __repr__(self):
+        CheckParent(self)
+        return "link on " + str(self.parent)
+
+    def __str__(self):
+        CheckParent(self)
+        return "link on " + str(self.parent)
+
+    def _border(self, doc, xref):
+        pdf = _as_pdf_document(doc, required=0)
+        if not pdf.m_internal:
+            return
+        link_obj = mupdf.pdf_new_indirect(pdf, xref, 0)
+        if not link_obj.m_internal:
+            return
+        b = JM_annot_border(link_obj)
+        return b
+
+    def _colors(self, doc, xref):
+        pdf = _as_pdf_document(doc, required=0)
+        if not pdf.m_internal:
+            return
+        link_obj = mupdf.pdf_new_indirect( pdf, xref, 0)
+        if not link_obj.m_internal:
+            raise ValueError( MSG_BAD_XREF)
+        b = JM_annot_colors( link_obj)
+        return b
+
+    def _erase(self):
+        self.parent = None
+        self.thisown = False
+
+    def _setBorder(self, border, doc, xref):
+        pdf = _as_pdf_document(doc, required=0)
+        if not pdf.m_internal:
+            return
+        link_obj = mupdf.pdf_new_indirect(pdf, xref, 0)
+        if not link_obj.m_internal:
+            return
+        b = JM_annot_set_border(border, pdf, link_obj)
+        return b
+        
+    @property
+    def border(self):
+        return self._border(self.parent.parent.this, self.xref)
+
+    @property
+    def colors(self):
+        return self._colors(self.parent.parent.this, self.xref)
+
+    @property
+    def dest(self):
+        """Create link destination details."""
+        if hasattr(self, "parent") and self.parent is None:
+            raise ValueError("orphaned object: parent is None")
+        if self.parent.parent.is_closed or self.parent.parent.is_encrypted:
+            raise ValueError("document closed or encrypted")
+        doc = self.parent.parent
+
+        if self.is_external or self.uri.startswith("#"):
+            uri = None
+        else:
+            uri = doc.resolve_link(self.uri)
+
+        return linkDest(self, uri, doc)
+
+    @property
+    def flags(self)->int:
+        CheckParent(self)
+        doc = self.parent.parent
+        if not doc.is_pdf:
+            return 0
+        f = doc.xref_get_key(self.xref, "F")
+        if f[1] != "null":
+            return int(f[1])
+        return 0
+
+    @property
+    def is_external(self):
+        """Flag the link as external."""
+        CheckParent(self)
+        if g_use_extra:
+            return extra.Link_is_external( self.this)
+        this_link = self.this
+        if not this_link.m_internal or not this_link.m_internal.uri:
+            return False
+        return bool( mupdf.fz_is_external_link( this_link.m_internal.uri))
+
+    @property
+    def next(self):
+        """Next link."""
+        if not self.this.m_internal:
+            return None
+        CheckParent(self)
+        if 0 and g_use_extra:
+            val = extra.Link_next( self.this)
+        else:
+            val = self.this.next()
+        if not val.m_internal:
+            return None
+        val = Link( val)
+        if val:
+            val.thisown = True
+            val.parent = self.parent  # copy owning page from prev link
+            val.parent._annot_refs[id(val)] = val
+            if self.xref > 0:  # prev link has an xref
+                link_xrefs = [x[0] for x in self.parent.annot_xrefs() if x[1] == mupdf.PDF_ANNOT_LINK]
+                link_ids = [x[2] for x in self.parent.annot_xrefs() if x[1] == mupdf.PDF_ANNOT_LINK]
+                idx = link_xrefs.index(self.xref)
+                val.xref = link_xrefs[idx + 1]
+                val.id = link_ids[idx + 1]
+            else:
+                val.xref = 0
+                val.id = ""
+        return val
+
+    @property
+    def rect(self):
+        """Rectangle ('hot area')."""
+        CheckParent(self)
+        # utils.py:getLinkDict() appears to expect exceptions from us, so we
+        # ensure that we raise on error.
+        if self.this is None or not self.this.m_internal:
+            raise Exception( 'self.this.m_internal not available')
+        val = JM_py_from_rect( self.this.rect())
+        val = Rect(val)
+        return val
+
+    def set_border(self, border=None, width=0, dashes=None, style=None):
+        if type(border) is not dict:
+            border = {"width": width, "style": style, "dashes": dashes}
+        return self._setBorder(border, self.parent.parent.this, self.xref)
+
+    def set_colors(self, colors=None, stroke=None, fill=None):
+        """Set border colors."""
+        CheckParent(self)
+        doc = self.parent.parent
+        if type(colors) is not dict:
+            colors = {"fill": fill, "stroke": stroke}
+        fill = colors.get("fill")
+        stroke = colors.get("stroke")
+        if fill is not None:
+            message("warning: links have no fill color")
+        if stroke in ([], ()):
+            doc.xref_set_key(self.xref, "C", "[]")
+            return
+        if hasattr(stroke, "__float__"):
+            stroke = [float(stroke)]
+        CheckColor(stroke)
+        assert len(stroke) in (1, 3, 4)
+        s = f"[{_format_g(stroke)}]"
+        doc.xref_set_key(self.xref, "C", s)
+
+    def set_flags(self, flags):
+        CheckParent(self)
+        doc = self.parent.parent
+        if not doc.is_pdf:
+            raise ValueError("is no PDF")
+        if not type(flags) is int:
+            raise ValueError("bad 'flags' value")
+        doc.xref_set_key(self.xref, "F", str(flags))
+        return None
+
+    @property
+    def uri(self):
+        """Uri string."""
+        #CheckParent(self)
+        if g_use_extra:
+            return extra.link_uri(self.this)
+        this_link = self.this
+        return this_link.m_internal.uri if this_link.m_internal else ''
+
+    page = -1
+
+
+class Matrix:
+
+    def __abs__(self):
+        return math.sqrt(sum([c*c for c in self]))
+
+    def __add__(self, m):
+        if hasattr(m, "__float__"):
+            return Matrix(self.a + m, self.b + m, self.c + m,
+                          self.d + m, self.e + m, self.f + m)
+        if len(m) != 6:
+            raise ValueError("Matrix: bad seq len")
+        return Matrix(self.a + m[0], self.b + m[1], self.c + m[2],
+                          self.d + m[3], self.e + m[4], self.f + m[5])
+
+    def __bool__(self):
+        return not (max(self) == min(self) == 0)
+
+    def __eq__(self, mat):
+        if not hasattr(mat, "__len__"):
+            return False
+        return len(mat) == 6 and not (self - mat)
+
+    def __getitem__(self, i):
+        return (self.a, self.b, self.c, self.d, self.e, self.f)[i]
+
+    def __init__(self, *args, a=None, b=None, c=None, d=None, e=None, f=None):
+        """
+        Matrix() - all zeros
+        Matrix(a, b, c, d, e, f)
+        Matrix(zoom-x, zoom-y) - zoom
+        Matrix(shear-x, shear-y, 1) - shear
+        Matrix(degree) - rotate
+        Matrix(Matrix) - new copy
+        Matrix(sequence) - from 'sequence'
+        Matrix(mupdf.FzMatrix) - from MuPDF class wrapper for fz_matrix.
+        
+        Explicit keyword args a, b, c, d, e, f override any earlier settings if
+        not None.
+        """
+        if not args:
+            self.a = self.b = self.c = self.d = self.e = self.f = 0.0
+        elif len(args) > 6:
+            raise ValueError("Matrix: bad seq len")
+        elif len(args) == 6:  # 6 numbers
+            self.a, self.b, self.c, self.d, self.e, self.f = map(float, args)
+        elif len(args) == 1:  # either an angle or a sequ
+            if isinstance(args[0], mupdf.FzMatrix):
+                self.a = args[0].a
+                self.b = args[0].b
+                self.c = args[0].c
+                self.d = args[0].d
+                self.e = args[0].e
+                self.f = args[0].f
+            elif hasattr(args[0], "__float__"):
+                theta = math.radians(args[0])
+                c_ = round(math.cos(theta), 8)
+                s_ = round(math.sin(theta), 8)
+                self.a = self.d = c_
+                self.b = s_
+                self.c = -s_
+                self.e = self.f = 0.0
+            else:
+                self.a, self.b, self.c, self.d, self.e, self.f = map(float, args[0])
+        elif len(args) == 2 or len(args) == 3 and args[2] == 0:
+            self.a, self.b, self.c, self.d, self.e, self.f = float(args[0]), \
+                0.0, 0.0, float(args[1]), 0.0, 0.0
+        elif len(args) == 3 and args[2] == 1:
+            self.a, self.b, self.c, self.d, self.e, self.f = 1.0, \
+                float(args[1]), float(args[0]), 1.0, 0.0, 0.0
+        else:
+            raise ValueError("Matrix: bad args")
+        
+        # Override with explicit args if specified.
+        if a is not None:   self.a = a
+        if b is not None:   self.b = b
+        if c is not None:   self.c = c
+        if d is not None:   self.d = d
+        if e is not None:   self.e = e
+        if f is not None:   self.f = f
+
+    def __invert__(self):
+        """Calculate inverted matrix."""
+        m1 = Matrix()
+        m1.invert(self)
+        return m1
+
+    def __len__(self):
+        return 6
+
+    def __mul__(self, m):
+        if hasattr(m, "__float__"):
+            return Matrix(self.a * m, self.b * m, self.c * m,
+                          self.d * m, self.e * m, self.f * m)
+        m1 = Matrix(1,1)
+        return m1.concat(self, m)
+
+    def __neg__(self):
+        return Matrix(-self.a, -self.b, -self.c, -self.d, -self.e, -self.f)
+
+    def __nonzero__(self):
+        return not (max(self) == min(self) == 0)
+
+    def __pos__(self):
+        return Matrix(self)
+
+    def __repr__(self):
+        return "Matrix" + str(tuple(self))
+
+    def __setitem__(self, i, v):
+        v = float(v)
+        if   i == 0: self.a = v
+        elif i == 1: self.b = v
+        elif i == 2: self.c = v
+        elif i == 3: self.d = v
+        elif i == 4: self.e = v
+        elif i == 5: self.f = v
+        else:
+            raise IndexError("index out of range")
+        return
+
+    def __sub__(self, m):
+        if hasattr(m, "__float__"):
+            return Matrix(self.a - m, self.b - m, self.c - m,
+                          self.d - m, self.e - m, self.f - m)
+        if len(m) != 6:
+            raise ValueError("Matrix: bad seq len")
+        return Matrix(self.a - m[0], self.b - m[1], self.c - m[2],
+                          self.d - m[3], self.e - m[4], self.f - m[5])
+
+    def __truediv__(self, m):
+        if hasattr(m, "__float__"):
+            return Matrix(self.a * 1./m, self.b * 1./m, self.c * 1./m,
+                          self.d * 1./m, self.e * 1./m, self.f * 1./m)
+        m1 = util_invert_matrix(m)[1]
+        if not m1:
+            raise ZeroDivisionError("matrix not invertible")
+        m2 = Matrix(1,1)
+        return m2.concat(self, m1)
+
+    def concat(self, one, two):
+        """Multiply two matrices and replace current one."""
+        if not len(one) == len(two) == 6:
+            raise ValueError("Matrix: bad seq len")
+        self.a, self.b, self.c, self.d, self.e, self.f = util_concat_matrix(one, two)
+        return self
+
+    def invert(self, src=None):
+        """Calculate the inverted matrix. Return 0 if successful and replace
+        current one. Else return 1 and do nothing.
+        """
+        if src is None:
+            dst = util_invert_matrix(self)
+        else:
+            dst = util_invert_matrix(src)
+        if dst[0] == 1:
+            return 1
+        self.a, self.b, self.c, self.d, self.e, self.f = dst[1]
+        return 0
+
+    @property
+    def is_rectilinear(self):
+        """True if rectangles are mapped to rectangles."""
+        return (abs(self.b) < EPSILON and abs(self.c) < EPSILON) or \
+            (abs(self.a) < EPSILON and abs(self.d) < EPSILON)
+
+    def prerotate(self, theta):
+        """Calculate pre rotation and replace current matrix."""
+        theta = float(theta)
+        while theta < 0: theta += 360
+        while theta >= 360: theta -= 360
+        if abs(0 - theta) < EPSILON:
+            pass
+
+        elif abs(90.0 - theta) < EPSILON:
+            a = self.a
+            b = self.b
+            self.a = self.c
+            self.b = self.d
+            self.c = -a
+            self.d = -b
+
+        elif abs(180.0 - theta) < EPSILON:
+            self.a = -self.a
+            self.b = -self.b
+            self.c = -self.c
+            self.d = -self.d
+
+        elif abs(270.0 - theta) < EPSILON:
+            a = self.a
+            b = self.b
+            self.a = -self.c
+            self.b = -self.d
+            self.c = a
+            self.d = b
+
+        else:
+            rad = math.radians(theta)
+            s = math.sin(rad)
+            c = math.cos(rad)
+            a = self.a
+            b = self.b
+            self.a = c * a + s * self.c
+            self.b = c * b + s * self.d
+            self.c =-s * a + c * self.c
+            self.d =-s * b + c * self.d
+
+        return self
+
+    def prescale(self, sx, sy):
+        """Calculate pre scaling and replace current matrix."""
+        sx = float(sx)
+        sy = float(sy)
+        self.a *= sx
+        self.b *= sx
+        self.c *= sy
+        self.d *= sy
+        return self
+
+    def preshear(self, h, v):
+        """Calculate pre shearing and replace current matrix."""
+        h = float(h)
+        v = float(v)
+        a, b = self.a, self.b
+        self.a += v * self.c
+        self.b += v * self.d
+        self.c += h * a
+        self.d += h * b
+        return self
+
+    def pretranslate(self, tx, ty):
+        """Calculate pre translation and replace current matrix."""
+        tx = float(tx)
+        ty = float(ty)
+        self.e += tx * self.a + ty * self.c
+        self.f += tx * self.b + ty * self.d
+        return self
+
+    __inv__ = __invert__
+    __div__ = __truediv__
+    norm = __abs__
+
+
+class IdentityMatrix(Matrix):
+    """Identity matrix [1, 0, 0, 1, 0, 0]"""
+
+    def __hash__(self):
+        return hash((1,0,0,1,0,0))
+
+    def __init__(self):
+        Matrix.__init__(self, 1.0, 1.0)
+
+    def __repr__(self):
+        return "IdentityMatrix(1.0, 0.0, 0.0, 1.0, 0.0, 0.0)"
+
+    def __setattr__(self, name, value):
+        if name in "ad":
+            self.__dict__[name] = 1.0
+        elif name in "bcef":
+            self.__dict__[name] = 0.0
+        else:
+            self.__dict__[name] = value
+
+    def checkargs(*args):
+        raise NotImplementedError("Identity is readonly")
+
+Identity = IdentityMatrix()
+
+
+class linkDest:
+    """link or outline destination details"""
+
+    def __init__(self, obj, rlink, document=None):
+        isExt = obj.is_external
+        isInt = not isExt
+        self.dest = ""
+        self.file_spec = ""
+        self.flags = 0
+        self.is_map = False
+        self.is_uri = False
+        self.kind = LINK_NONE
+        self.lt = Point(0, 0)
+        self.named = dict()
+        self.new_window = ""
+        self.page = obj.page
+        self.rb = Point(0, 0)
+        self.uri = obj.uri
+        
+        def uri_to_dict(uri):
+            items = self.uri[1:].split('&')
+            ret = dict()
+            for item in items:
+                eq = item.find('=')
+                if eq >= 0:
+                    ret[item[:eq]] = item[eq+1:]
+                else:
+                    ret[item] = None
+            return ret
+
+        def unescape(name):
+            """Unescape '%AB' substrings to chr(0xAB)."""
+            split = name.replace("%%", "%25")  # take care of escaped '%'
+            split = split.split("%")
+            newname = split[0]
+            for item in split[1:]:
+                piece = item[:2]
+                newname += chr(int(piece, base=16))
+                newname += item[2:]
+            return newname
+        
+        if rlink and not self.uri.startswith("#"):
+            self.uri = f"#page={rlink[0] + 1}&zoom=0,{_format_g(rlink[1])},{_format_g(rlink[2])}"
+        if obj.is_external:
+            self.page = -1
+            self.kind = LINK_URI
+        if not self.uri:
+            self.page = -1
+            self.kind = LINK_NONE
+        if isInt and self.uri:
+            self.uri = self.uri.replace("&zoom=nan", "&zoom=0")
+            if self.uri.startswith("#"):
+                self.kind = LINK_GOTO
+                m = re.match('^#page=([0-9]+)&zoom=([0-9.]+),(-?[0-9.]+),(-?[0-9.]+)$', self.uri)
+                if m:
+                    self.page = int(m.group(1)) - 1
+                    self.lt = Point(float((m.group(3))), float(m.group(4)))
+                    self.flags = self.flags | LINK_FLAG_L_VALID | LINK_FLAG_T_VALID
+                else:
+                    m = re.match('^#page=([0-9]+)$', self.uri)
+                    if m:
+                        self.page = int(m.group(1)) - 1
+                    else:
+                        self.kind = LINK_NAMED
+                        m = re.match('^#nameddest=(.*)', self.uri)
+                        assert document
+                        if document and m:
+                            named = unescape(m.group(1))
+                            self.named = document.resolve_names().get(named)
+                            if self.named is None:
+                                # document.resolve_names() does not contain an
+                                # entry for `named` so use an empty dict.
+                                self.named = dict()
+                            self.named['nameddest'] = named
+                        else:
+                            self.named = uri_to_dict(self.uri[1:])
+            else:
+                self.kind = LINK_NAMED
+                self.named = uri_to_dict(self.uri)
+        if obj.is_external:
+            if not self.uri:
+                pass
+            elif self.uri.startswith("file:"):
+                self.file_spec = self.uri[5:]
+                if self.file_spec.startswith("//"):
+                    self.file_spec = self.file_spec[2:]
+                self.is_uri = False
+                self.uri = ""
+                self.kind = LINK_LAUNCH
+                ftab = self.file_spec.split("#")
+                if len(ftab) == 2:
+                    if ftab[1].startswith("page="):
+                        self.kind = LINK_GOTOR
+                        self.file_spec = ftab[0]
+                        self.page = int(ftab[1].split("&")[0][5:]) - 1
+            elif ":" in self.uri:
+                self.is_uri = True
+                self.kind = LINK_URI
+            else:
+                self.is_uri = True
+                self.kind = LINK_LAUNCH
+        assert isinstance(self.named, dict)
+
+class Widget:
+    '''
+    Class describing a PDF form field ("widget")
+    '''
+
+    def __init__(self):
+        self.border_color = None
+        self.border_style = "S"
+        self.border_width = 0
+        self.border_dashes = None
+        self.choice_values = None  # choice fields only
+        self.rb_parent = None   # radio buttons only: xref of owning parent
+
+        self.field_name = None  # field name
+        self.field_label = None # field label
+        self.field_value = None
+        self.field_flags = 0
+        self.field_display = 0
+        self.field_type = 0  # valid range 1 through 7
+        self.field_type_string = None  # field type as string
+
+        self.fill_color = None
+        self.button_caption = None  # button caption
+        self.is_signed = None  # True / False if signature
+        self.text_color = (0, 0, 0)
+        self.text_font = "Helv"
+        self.text_fontsize = 0
+        self.text_maxlen = 0  # text fields only
+        self.text_format = 0  # text fields only
+        self._text_da = ""  # /DA = default appearance
+
+        self.script = None  # JavaScript (/A)
+        self.script_stroke = None  # JavaScript (/AA/K)
+        self.script_format = None  # JavaScript (/AA/F)
+        self.script_change = None  # JavaScript (/AA/V)
+        self.script_calc = None  # JavaScript (/AA/C)
+        self.script_blur = None  # JavaScript (/AA/Bl)
+        self.script_focus = None  # JavaScript (/AA/Fo) codespell:ignore
+
+        self.rect = None  # annot value
+        self.xref = 0  # annot value
+
+    def __repr__(self):
+        #return "'%s' widget on %s" % (self.field_type_string, str(self.parent))
+        # No self.parent.
+        return f'Widget:(field_type={self.field_type_string} script={self.script})'
+        return "'%s' widget" % (self.field_type_string)
+
+    def _adjust_font(self):
+        """Ensure text_font is from our list and correctly spelled.
+        """
+        if not self.text_font:
+            self.text_font = "Helv"
+            return
+        valid_fonts = ("Cour", "TiRo", "Helv", "ZaDb")
+        for f in valid_fonts:
+            if self.text_font.lower() == f.lower():
+                self.text_font = f
+                return
+        self.text_font = "Helv"
+        return
+
+    def _checker(self):
+        """Any widget type checks.
+        """
+        if self.field_type not in range(1, 8):
+            raise ValueError("bad field type")
+
+        # if setting a radio button to ON, first set Off all buttons
+        # in the group - this is not done by MuPDF:
+        if self.field_type == mupdf.PDF_WIDGET_TYPE_RADIOBUTTON and self.field_value not in (False, "Off") and hasattr(self, "parent"):
+            # so we are about setting this button to ON/True
+            # check other buttons in same group and set them to 'Off'
+            doc = self.parent.parent
+            kids_type, kids_value = doc.xref_get_key(self.xref, "Parent/Kids")
+            if kids_type == "array":
+                xrefs = tuple(map(int, kids_value[1:-1].replace("0 R","").split()))
+                for xref in xrefs:
+                    if xref != self.xref:
+                        doc.xref_set_key(xref, "AS", "/Off")
+        # the calling method will now set the intended button to on and
+        # will find everything prepared for correct functioning.
+
+    def _parse_da(self):
+        """Extract font name, size and color from default appearance string (/DA object).
+
+        Equivalent to 'pdf_parse_default_appearance' function in MuPDF's 'pdf-annot.c'.
+        """
+        if not self._text_da:
+            return
+        font = "Helv"
+        fsize = 0
+        col = (0, 0, 0)
+        dat = self._text_da.split()  # split on any whitespace
+        for i, item in enumerate(dat):
+            if item == "Tf":
+                font = dat[i - 2][1:]
+                fsize = float(dat[i - 1])
+                dat[i] = dat[i-1] = dat[i-2] = ""
+                continue
+            if item == "g":  # unicolor text
+                col = [(float(dat[i - 1]))]
+                dat[i] = dat[i-1] = ""
+                continue
+            if item == "rg":  # RGB colored text
+                col = [float(f) for f in dat[i - 3:i]]
+                dat[i] = dat[i-1] = dat[i-2] = dat[i-3] = ""
+                continue
+        self.text_font = font
+        self.text_fontsize = fsize
+        self.text_color = col
+        self._text_da = ""
+        return
+
+    def _validate(self):
+        """Validate the class entries.
+        """
+        if (self.rect.is_infinite
+            or self.rect.is_empty
+           ):
+            raise ValueError("bad rect")
+
+        if not self.field_name:
+            raise ValueError("field name missing")
+
+        if self.field_label == "Unnamed":
+            self.field_label = None
+        CheckColor(self.border_color)
+        CheckColor(self.fill_color)
+        if not self.text_color:
+            self.text_color = (0, 0, 0)
+        CheckColor(self.text_color)
+
+        if not self.border_width:
+            self.border_width = 0
+
+        if not self.text_fontsize:
+            self.text_fontsize = 0
+
+        self.border_style = self.border_style.upper()[0:1]
+
+        # standardize content of JavaScript entries
+        btn_type = self.field_type in (
+                mupdf.PDF_WIDGET_TYPE_BUTTON,
+                mupdf.PDF_WIDGET_TYPE_CHECKBOX,
+                mupdf.PDF_WIDGET_TYPE_RADIOBUTTON,
+                )
+        if not self.script:
+            self.script = None
+        elif type(self.script) is not str:
+            raise ValueError("script content must be a string")
+
+        # buttons cannot have the following script actions
+        if btn_type or not self.script_calc:
+            self.script_calc = None
+        elif type(self.script_calc) is not str:
+            raise ValueError("script_calc content must be a string")
+
+        if btn_type or not self.script_change:
+            self.script_change = None
+        elif type(self.script_change) is not str:
+            raise ValueError("script_change content must be a string")
+
+        if btn_type or not self.script_format:
+            self.script_format = None
+        elif type(self.script_format) is not str:
+            raise ValueError("script_format content must be a string")
+
+        if btn_type or not self.script_stroke:
+            self.script_stroke = None
+        elif type(self.script_stroke) is not str:
+            raise ValueError("script_stroke content must be a string")
+
+        if btn_type or not self.script_blur:
+            self.script_blur = None
+        elif type(self.script_blur) is not str:
+            raise ValueError("script_blur content must be a string")
+
+        if btn_type or not self.script_focus:
+            self.script_focus = None
+        elif type(self.script_focus) is not str:
+            raise ValueError("script_focus content must be a string")
+
+        self._checker()  # any field_type specific checks
+
+    def _sync_flags(self):
+        """Propagate the field flags.
+
+        If this widget has a "/Parent", set its field flags and that of all
+        its /Kids widgets to the value of the current widget.
+        Only possible for widgets existing in the PDF.
+
+        Returns True or False.
+        """
+        if not self.xref:
+            return False  # no xref: widget not in the PDF
+        doc = self.parent.parent  # the owning document
+        assert doc
+        pdf = _as_pdf_document(doc)
+        # load underlying PDF object
+        pdf_widget = mupdf.pdf_load_object(pdf, self.xref)
+        Parent = mupdf.pdf_dict_get(pdf_widget, PDF_NAME("Parent"))
+        if not Parent.pdf_is_dict():
+            return False  # no /Parent: nothing to do
+
+        # put the field flags value into the parent field flags:
+        Parent.pdf_dict_put_int(PDF_NAME("Ff"), self.field_flags)
+
+        # also put that value into all kids of the Parent
+        kids = Parent.pdf_dict_get(PDF_NAME("Kids"))
+        if not kids.pdf_is_array():
+            message("warning: malformed PDF, Parent has no Kids array")
+            return False  # no /Kids: should never happen!
+
+        for i in range(kids.pdf_array_len()):  # walk through all kids
+            # access kid widget, and do some precautionary checks
+            kid = kids.pdf_array_get(i)
+            if not kid.pdf_is_dict():
+                continue
+            xref = kid.pdf_to_num()  # get xref of the kid
+            if xref == self.xref:  # skip self widget
+                continue
+            subtype = kid.pdf_dict_get(PDF_NAME("Subtype"))
+            if not subtype.pdf_to_name() == "Widget":
+                continue
+            # put the field flags value into the kid field flags:
+            kid.pdf_dict_put_int(PDF_NAME("Ff"), self.field_flags)
+
+        return True  # all done
+
+    def button_states(self):
+        """Return the on/off state names for button widgets.
+
+        A button may have 'normal' or 'pressed down' appearances. While the 'Off'
+        state is usually called like this, the 'On' state is often given a name
+        relating to the functional context.
+        """
+        if self.field_type not in (2, 5):
+            return None  # no button type
+        if hasattr(self, "parent"):  # field already exists on page
+            doc = self.parent.parent
+        else:
+            return
+        xref = self.xref
+        states = {"normal": None, "down": None}
+        APN = doc.xref_get_key(xref, "AP/N")
+        if APN[0] == "dict":
+            nstates = []
+            APN = APN[1][2:-2]
+            apnt = APN.split("/")[1:]
+            for x in apnt:
+                nstates.append(x.split()[0])
+            states["normal"] = nstates
+        if APN[0] == "xref":
+            nstates = []
+            nxref = int(APN[1].split(" ")[0])
+            APN = doc.xref_object(nxref)
+            apnt = APN.split("/")[1:]
+            for x in apnt:
+                nstates.append(x.split()[0])
+            states["normal"] = nstates
+        APD = doc.xref_get_key(xref, "AP/D")
+        if APD[0] == "dict":
+            dstates = []
+            APD = APD[1][2:-2]
+            apdt = APD.split("/")[1:]
+            for x in apdt:
+                dstates.append(x.split()[0])
+            states["down"] = dstates
+        if APD[0] == "xref":
+            dstates = []
+            dxref = int(APD[1].split(" ")[0])
+            APD = doc.xref_object(dxref)
+            apdt = APD.split("/")[1:]
+            for x in apdt:
+                dstates.append(x.split()[0])
+            states["down"] = dstates
+        return states
+
+    @property
+    def next(self):
+        return self._annot.next
+
+    def on_state(self):
+        """Return the "On" value for button widgets.
+        
+        This is useful for radio buttons mainly. Checkboxes will always return
+        "Yes". Radio buttons will return the string that is unequal to "Off"
+        as returned by method button_states().
+        If the radio button is new / being created, it does not yet have an
+        "On" value. In this case, a warning is shown and True is returned.
+        """
+        if self.field_type not in (2, 5):
+            return None  # no checkbox or radio button
+        bstate = self.button_states()
+        if bstate is None:
+            bstate = dict()
+        for k in bstate.keys():
+            for v in bstate[k]:
+                if v != "Off":
+                    return v
+        message("warning: radio button has no 'On' value.")
+        return True
+
+    def reset(self):
+        """Reset the field value to its default.
+        """
+        TOOLS._reset_widget(self._annot)
+
+    def update(self, sync_flags=False):
+        """Reflect Python object in the PDF."""
+        self._validate()
+
+        self._adjust_font()  # ensure valid text_font name
+
+        # now create the /DA string
+        self._text_da = ""
+        if   len(self.text_color) == 3:
+            fmt = "{:g} {:g} {:g} rg /{f:s} {s:g} Tf" + self._text_da
+        elif len(self.text_color) == 1:
+            fmt = "{:g} g /{f:s} {s:g} Tf" + self._text_da
+        elif len(self.text_color) == 4:
+            fmt = "{:g} {:g} {:g} {:g} k /{f:s} {s:g} Tf" + self._text_da
+        self._text_da = fmt.format(*self.text_color, f=self.text_font,
+                                    s=self.text_fontsize)
+        # finally update the widget
+
+        # if widget has a '/AA/C' script, make sure it is in the '/CO'
+        # array of the '/AcroForm' dictionary.
+        if self.script_calc:  # there is a "calculation" script:
+            # make sure we are in the /CO array
+            util_ensure_widget_calc(self._annot)
+
+        # finally update the widget
+        TOOLS._save_widget(self._annot, self)
+        self._text_da = ""
+        if sync_flags:
+            self._sync_flags()  # propagate field flags to parent and kids
+
+
+from . import _extra
+
+
+class Outline:
+
+    def __init__(self, ol):
+        self.this = ol
+
+    @property
+    def dest(self):
+        '''outline destination details'''
+        return linkDest(self, None, None)
+
+    def destination(self, document):
+        '''
+        Like `dest` property but uses `document` to resolve destinations for
+        kind=LINK_NAMED.
+        '''
+        return linkDest(self, None, document)
+        
+    @property
+    def down(self):
+        ol = self.this
+        down_ol = ol.down()
+        if not down_ol.m_internal:
+            return
+        return Outline(down_ol)
+
+    @property
+    def is_external(self):
+        if g_use_extra:
+            # calling _extra.* here appears to save significant time in
+            # test_toc.py:test_full_toc, 1.2s=>0.94s.
+            #
+            return _extra.Outline_is_external( self.this)
+        ol = self.this
+        if not ol.m_internal:
+            return False
+        uri = ol.m_internal.uri if 1 else ol.uri()
+        if uri is None:
+            return False
+        return mupdf.fz_is_external_link(uri)
+
+    @property
+    def is_open(self):
+        if 1:
+            return self.this.m_internal.is_open
+        return self.this.is_open()
+
+    @property
+    def next(self):
+        ol = self.this
+        next_ol = ol.next()
+        if not next_ol.m_internal:
+            return
+        return Outline(next_ol)
+
+    @property
+    def page(self):
+        if 1:
+            return self.this.m_internal.page.page
+        return self.this.page().page
+
+    @property
+    def title(self):
+        return self.this.m_internal.title
+
+    @property
+    def uri(self):
+        ol = self.this
+        if not ol.m_internal:
+            return None
+        return ol.m_internal.uri
+
+    @property
+    def x(self):
+        return self.this.m_internal.x
+
+    @property
+    def y(self):
+        return self.this.m_internal.y
+
+    __slots__ = [ 'this']
+
+
+def _make_PdfFilterOptions(
+        recurse=0,
+        instance_forms=0,
+        ascii=0,
+        no_update=0,
+        sanitize=0,
+        sopts=None,
+        ):
+    '''
+    Returns a mupdf.PdfFilterOptions instance.
+    '''
+
+    filter_ = mupdf.PdfFilterOptions()
+    filter_.recurse = recurse
+    filter_.instance_forms = instance_forms
+    filter_.ascii = ascii
+    
+    filter_.no_update = no_update
+    if sanitize:
+        # We want to use a PdfFilterFactory whose `.filter` fn pointer is
+        # set to MuPDF's `pdf_new_sanitize_filter()`. But not sure how to
+        # get access to this raw fn in Python; and on Windows raw MuPDF
+        # functions are not even available to C++.
+        #
+        # So we use SWIG Director to implement our own
+        # PdfFilterFactory whose `filter()` method calls
+        # `mupdf.ll_pdf_new_sanitize_filter()`.
+        if sopts:
+            assert isinstance(sopts, mupdf.PdfSanitizeFilterOptions)
+        else:
+            sopts = mupdf.PdfSanitizeFilterOptions()
+        class Factory(mupdf.PdfFilterFactory2):
+            def __init__(self):
+                super().__init__()
+                self.use_virtual_filter()
+                self.sopts = sopts
+            def filter(self, ctx, doc, chain, struct_parents, transform, options):
+                if 0:
+                    log(f'sanitize filter.filter():')
+                    log(f'    {self=}')
+                    log(f'    {ctx=}')
+                    log(f'    {doc=}')
+                    log(f'    {chain=}')
+                    log(f'    {struct_parents=}')
+                    log(f'    {transform=}')
+                    log(f'    {options=}')
+                    log(f'    {self.sopts.internal()=}')
+                return mupdf.ll_pdf_new_sanitize_filter(
+                        doc,
+                        chain,
+                        struct_parents,
+                        transform,
+                        options,
+                        self.sopts.internal(),
+                        )
+
+        factory = Factory()
+        filter_.add_factory(factory.internal())
+        filter_._factory = factory
+    return filter_
+
+
+class Page:
+
+    def __init__(self, page, document):
+        assert isinstance(page, (mupdf.FzPage, mupdf.PdfPage)), f'page is: {page}'
+        self.this = page
+        self.thisown = True
+        self.last_point = None
+        self.draw_cont = ''
+        self._annot_refs = dict()
+        self.parent = document
+        if page.m_internal:
+            if isinstance( page, mupdf.PdfPage):
+                self.number = page.m_internal.super.number
+            else:
+                self.number = page.m_internal.number
+        else:
+            self.number = None
+
+    def __repr__(self):
+        return self.__str__()
+        CheckParent(self)
+        x = self.parent.name
+        if self.parent.stream is not None:
+            x = "" % (self.parent._graft_id,)
+        if x == "":
+            x = "" % self.parent._graft_id
+        return "page %s of %s" % (self.number, x)
+
+    def __str__(self):
+        #CheckParent(self)
+        parent = getattr(self, 'parent', None)
+        if isinstance(self.this.m_internal, mupdf.pdf_page):
+            number = self.this.m_internal.super.number
+        else:
+            number = self.this.m_internal.number
+        ret = f'page {number}'
+        if parent:
+            x = self.parent.name
+            if self.parent.stream is not None:
+                x = "" % (self.parent._graft_id,)
+            if x == "":
+                x = "" % self.parent._graft_id
+            ret += f' of {x}'
+        return ret
+
+    def _add_caret_annot(self, point):
+        if g_use_extra:
+            annot = extra._add_caret_annot( self.this, JM_point_from_py(point))
+        else:
+            page = self._pdf_page()
+            annot = mupdf.pdf_create_annot(page, mupdf.PDF_ANNOT_CARET)
+            if point:
+                p = JM_point_from_py(point)
+                r = mupdf.pdf_annot_rect(annot)
+                r = mupdf.FzRect(p.x, p.y, p.x + r.x1 - r.x0, p.y + r.y1 - r.y0)
+                mupdf.pdf_set_annot_rect(annot, r)
+            mupdf.pdf_update_annot(annot)
+            JM_add_annot_id(annot, "A")
+        return annot
+
+    def _add_file_annot(self, point, buffer_, filename, ufilename=None, desc=None, icon=None):
+        page = self._pdf_page()
+        uf = ufilename if ufilename else filename
+        d = desc if desc else filename
+        p = JM_point_from_py(point)
+        filebuf = JM_BufferFromBytes(buffer_)
+        if not filebuf.m_internal:
+            raise TypeError( MSG_BAD_BUFFER)
+        annot = mupdf.pdf_create_annot(page, mupdf.PDF_ANNOT_FILE_ATTACHMENT)
+        r = mupdf.pdf_annot_rect(annot)
+        r = mupdf.fz_make_rect(p.x, p.y, p.x + r.x1 - r.x0, p.y + r.y1 - r.y0)
+        mupdf.pdf_set_annot_rect(annot, r)
+        flags = mupdf.PDF_ANNOT_IS_PRINT
+        mupdf.pdf_set_annot_flags(annot, flags)
+
+        if icon:
+            mupdf.pdf_set_annot_icon_name(annot, icon)
+
+        val = JM_embed_file(page.doc(), filebuf, filename, uf, d, 1)
+        mupdf.pdf_dict_put(mupdf.pdf_annot_obj(annot), PDF_NAME('FS'), val)
+        mupdf.pdf_dict_put_text_string(mupdf.pdf_annot_obj(annot), PDF_NAME('Contents'), filename)
+        mupdf.pdf_update_annot(annot)
+        mupdf.pdf_set_annot_rect(annot, r)
+        mupdf.pdf_set_annot_flags(annot, flags)
+        JM_add_annot_id(annot, "A")
+        return Annot(annot)
+
+    def _add_freetext_annot(
+            self, rect,
+            text,
+            fontsize=11,
+            fontname=None,
+            text_color=None,
+            fill_color=None,
+            border_color=None,
+            border_width=0,
+            dashes=None,
+            callout=None,
+            line_end=mupdf.PDF_ANNOT_LE_OPEN_ARROW,
+            opacity=1,
+            align=0,
+            rotate=0,
+            richtext=False,
+            style=None,
+            ):
+        rc = f"""
+            
+            {text}"""
+        page = self._pdf_page()
+        if border_color and not richtext:
+            raise ValueError("cannot set border_color if rich_text is False")
+        if border_color and not text_color:
+            text_color = border_color
+        nfcol, fcol = JM_color_FromSequence(fill_color)
+        ntcol, tcol = JM_color_FromSequence(text_color)
+        r = JM_rect_from_py(rect)
+        if mupdf.fz_is_infinite_rect(r) or mupdf.fz_is_empty_rect(r):
+            raise ValueError( MSG_BAD_RECT)
+        annot = mupdf.pdf_create_annot(page, mupdf.PDF_ANNOT_FREE_TEXT)
+        annot_obj = mupdf.pdf_annot_obj(annot)
+
+        #insert text as 'contents' or 'RC' depending on 'richtext'
+        if not richtext:
+            mupdf.pdf_set_annot_contents(annot, text)
+        else:
+            mupdf.pdf_dict_put_text_string(annot_obj,PDF_NAME("RC"), rc)
+            if style:
+                mupdf.pdf_dict_put_text_string(annot_obj,PDF_NAME("DS"), style)
+
+        mupdf.pdf_set_annot_rect(annot, r)
+
+        while rotate < 0:
+            rotate += 360
+        while rotate >= 360:
+            rotate -= 360
+        if rotate != 0:
+            mupdf.pdf_dict_put_int(annot_obj, PDF_NAME('Rotate'), rotate)
+
+        mupdf.pdf_set_annot_quadding(annot, align)
+
+        if nfcol > 0:
+            mupdf.pdf_set_annot_color(annot, fcol[:nfcol])
+
+        mupdf.pdf_set_annot_border_width(annot, border_width)
+        mupdf.pdf_set_annot_opacity(annot, opacity)
+        if dashes:
+            for d in dashes:
+                mupdf.pdf_add_annot_border_dash_item(annot, float(d))
+
+        # Insert callout information
+        if callout:
+            mupdf.pdf_dict_put(annot_obj, PDF_NAME("IT"), PDF_NAME("FreeTextCallout"))
+            mupdf.pdf_set_annot_callout_style(annot, line_end)
+            point_count = len(callout)
+            extra.JM_set_annot_callout_line(annot, tuple(callout), point_count)
+
+        # insert the default appearance string
+        if not richtext:
+            JM_make_annot_DA(annot, ntcol, tcol, fontname, fontsize)
+
+        mupdf.pdf_update_annot(annot)
+        JM_add_annot_id(annot, "A")
+        val = Annot(annot)
+        return val
+
+    def _add_ink_annot(self, list):
+        page = _as_pdf_page(self.this)
+        if not PySequence_Check(list):
+            raise ValueError( MSG_BAD_ARG_INK_ANNOT)
+        ctm = mupdf.FzMatrix()
+        mupdf.pdf_page_transform(page, mupdf.FzRect(0), ctm)
+        inv_ctm = mupdf.fz_invert_matrix(ctm)
+        annot = mupdf.pdf_create_annot(page, mupdf.PDF_ANNOT_INK)
+        annot_obj = mupdf.pdf_annot_obj(annot)
+        n0 = len(list)
+        inklist = mupdf.pdf_new_array(page.doc(), n0)
+
+        for j in range(n0):
+            sublist = list[j]
+            n1 = len(sublist)
+            stroke = mupdf.pdf_new_array(page.doc(), 2 * n1)
+
+            for i in range(n1):
+                p = sublist[i]
+                if not PySequence_Check(p) or PySequence_Size(p) != 2:
+                    raise ValueError( MSG_BAD_ARG_INK_ANNOT)
+                point = mupdf.fz_transform_point(JM_point_from_py(p), inv_ctm)
+                mupdf.pdf_array_push_real(stroke, point.x)
+                mupdf.pdf_array_push_real(stroke, point.y)
+
+            mupdf.pdf_array_push(inklist, stroke)
+
+        mupdf.pdf_dict_put(annot_obj, PDF_NAME('InkList'), inklist)
+        mupdf.pdf_update_annot(annot)
+        JM_add_annot_id(annot, "A")
+        return Annot(annot)
+
+    def _add_line_annot(self, p1, p2):
+        page = self._pdf_page()
+        annot = mupdf.pdf_create_annot(page, mupdf.PDF_ANNOT_LINE)
+        a = JM_point_from_py(p1)
+        b = JM_point_from_py(p2)
+        mupdf.pdf_set_annot_line(annot, a, b)
+        mupdf.pdf_update_annot(annot)
+        JM_add_annot_id(annot, "A")
+        assert annot.m_internal
+        return Annot(annot)
+
+    def _add_multiline(self, points, annot_type):
+        page = self._pdf_page()
+        if len(points) < 2:
+            raise ValueError( MSG_BAD_ARG_POINTS)
+        annot = mupdf.pdf_create_annot(page, annot_type)
+        for p in points:
+            if (PySequence_Size(p) != 2):
+                raise ValueError( MSG_BAD_ARG_POINTS)
+            point = JM_point_from_py(p)
+            mupdf.pdf_add_annot_vertex(annot, point)
+
+        mupdf.pdf_update_annot(annot)
+        JM_add_annot_id(annot, "A")
+        return Annot(annot)
+
+    def _add_redact_annot(self, quad, text=None, da_str=None, align=0, fill=None, text_color=None):
+        page = self._pdf_page()
+        fcol = [ 1, 1, 1, 0]
+        nfcol = 0
+        annot = mupdf.pdf_create_annot(page, mupdf.PDF_ANNOT_REDACT)
+        q = JM_quad_from_py(quad)
+        r = mupdf.fz_rect_from_quad(q)
+        # TODO calculate de-rotated rect
+        mupdf.pdf_set_annot_rect(annot, r)
+        if fill:
+            nfcol, fcol = JM_color_FromSequence(fill)
+            arr = mupdf.pdf_new_array(page.doc(), nfcol)
+            for i in range(nfcol):
+                mupdf.pdf_array_push_real(arr, fcol[i])
+            mupdf.pdf_dict_put(mupdf.pdf_annot_obj(annot), PDF_NAME('IC'), arr)
+        if text:
+            assert da_str
+            mupdf.pdf_dict_puts(
+                    mupdf.pdf_annot_obj(annot),
+                    "OverlayText",
+                    mupdf.pdf_new_text_string(text),
+                    )
+            mupdf.pdf_dict_put_text_string(mupdf.pdf_annot_obj(annot), PDF_NAME('DA'), da_str)
+            mupdf.pdf_dict_put_int(mupdf.pdf_annot_obj(annot), PDF_NAME('Q'), align)
+        mupdf.pdf_update_annot(annot)
+        JM_add_annot_id(annot, "A")
+        annot = mupdf.ll_pdf_keep_annot(annot.m_internal)
+        annot = mupdf.PdfAnnot( annot)
+        return Annot(annot)
+
+    def _add_square_or_circle(self, rect, annot_type):
+        page = self._pdf_page()
+        r = JM_rect_from_py(rect)
+        if mupdf.fz_is_infinite_rect(r) or mupdf.fz_is_empty_rect(r):
+            raise ValueError( MSG_BAD_RECT)
+        annot = mupdf.pdf_create_annot(page, annot_type)
+        mupdf.pdf_set_annot_rect(annot, r)
+        mupdf.pdf_update_annot(annot)
+        JM_add_annot_id(annot, "A")
+        assert annot.m_internal
+        return Annot(annot)
+
+    def _add_stamp_annot(self, rect, stamp=0):
+        rect = Rect(rect)
+        r = JM_rect_from_py(rect)
+        if mupdf.fz_is_infinite_rect(r) or mupdf.fz_is_empty_rect(r):
+            raise ValueError(MSG_BAD_RECT)
+        page = self._pdf_page()
+        stamp_id = [
+                "Approved",
+                "AsIs",
+                "Confidential",
+                "Departmental",
+                "Experimental",
+                "Expired",
+                "Final",
+                "ForComment",
+                "ForPublicRelease",
+                "NotApproved",
+                "NotForPublicRelease",
+                "Sold",
+                "TopSecret",
+                "Draft",
+                ]
+        n = len(stamp_id)
+        buf = None
+        name = None
+        if stamp in range(n):
+            name = stamp_id[stamp]
+        elif isinstance(stamp, Pixmap):
+            buf = stamp.tobytes()
+        elif isinstance(stamp, str):
+            buf = pathlib.Path(stamp).read_bytes()
+        elif isinstance(stamp, (bytes, bytearray)):
+            buf = stamp
+        elif isinstance(stamp, io.BytesIO):
+            buf = stamp.getvalue()
+        else:
+            name = stamp_id[0]
+
+        annot = mupdf.pdf_create_annot(page, mupdf.PDF_ANNOT_STAMP)
+        if buf:  # image stamp
+            fzbuff = mupdf.fz_new_buffer_from_copied_data(buf)
+            img = mupdf.fz_new_image_from_buffer(fzbuff)
+
+            # compute image boundary box on page
+            w, h = img.w(), img.h()
+            scale = min(rect.width / w, rect.height / h)
+            width = w * scale  # bbox width
+            height = h * scale  # bbox height
+
+            # center of "rect"
+            center = (rect.tl + rect.br) / 2
+            x0 = center.x - width / 2
+            y0 = center.y - height / 2
+            x1 = x0 + width
+            y1 = y0 + height
+            r = mupdf.fz_make_rect(x0, y0, x1, y1)
+            mupdf.pdf_set_annot_rect(annot, r)
+            mupdf.pdf_set_annot_stamp_image(annot, img)
+            mupdf.pdf_dict_put(mupdf.pdf_annot_obj(annot), PDF_NAME("Name"), mupdf.pdf_new_name("ImageStamp"))
+            mupdf.pdf_set_annot_contents(annot, "Image Stamp")
+        else:  # text stamp
+            mupdf.pdf_set_annot_rect(annot, r)
+            mupdf.pdf_dict_put(mupdf.pdf_annot_obj(annot), PDF_NAME("Name"), PDF_NAME(name))
+            mupdf.pdf_set_annot_contents(annot, name)
+        mupdf.pdf_update_annot(annot)
+        JM_add_annot_id(annot, "A")
+        return Annot(annot)
+
+    def _add_text_annot(self, point, text, icon=None):
+        page = self._pdf_page()
+        p = JM_point_from_py( point)
+        annot = mupdf.pdf_create_annot(page, mupdf.PDF_ANNOT_TEXT)
+        r = mupdf.pdf_annot_rect(annot)
+        r = mupdf.fz_make_rect(p.x, p.y, p.x + r.x1 - r.x0, p.y + r.y1 - r.y0)
+        mupdf.pdf_set_annot_rect(annot, r)
+        mupdf.pdf_set_annot_contents(annot, text)
+        if icon:
+            mupdf.pdf_set_annot_icon_name(annot, icon)
+        mupdf.pdf_update_annot(annot)
+        JM_add_annot_id(annot, "A")
+        return Annot(annot)
+
+    def _add_text_marker(self, quads, annot_type):
+
+        CheckParent(self)
+        if not self.parent.is_pdf:
+            raise ValueError("is no PDF")
+
+        val = Page__add_text_marker(self, quads, annot_type)
+        if not val:
+            return None
+        val.parent = weakref.proxy(self)
+        self._annot_refs[id(val)] = val
+
+        return val
+
+    def _addAnnot_FromString(self, linklist):
+        """Add links from list of object sources."""
+        CheckParent(self)
+        if g_use_extra:
+            self.__class__._addAnnot_FromString = extra.Page_addAnnot_FromString
+            #log('Page._addAnnot_FromString() deferring to extra.Page_addAnnot_FromString().')
+            return extra.Page_addAnnot_FromString( self.this, linklist)
+        page = _as_pdf_page(self.this)
+        lcount = len(linklist)  # link count
+        if lcount < 1:
+            return
+        i = -1
+
+        # insert links from the provided sources
+        if not isinstance(linklist, tuple):
+            raise ValueError( "bad 'linklist' argument")
+        if not mupdf.pdf_dict_get( page.obj(), PDF_NAME('Annots')).m_internal:
+            mupdf.pdf_dict_put_array( page.obj(), PDF_NAME('Annots'), lcount)
+        annots = mupdf.pdf_dict_get( page.obj(), PDF_NAME('Annots'))
+        assert annots.m_internal, f'{lcount=} {annots.m_internal=}'
+        for i in range(lcount):
+            txtpy = linklist[i]
+            text = JM_StrAsChar(txtpy)
+            if not text:
+                message("skipping bad link / annot item %i.", i)
+                continue
+            try:
+                annot = mupdf.pdf_add_object( page.doc(), JM_pdf_obj_from_str( page.doc(), text))
+                ind_obj = mupdf.pdf_new_indirect( page.doc(), mupdf.pdf_to_num( annot), 0)
+                mupdf.pdf_array_push( annots, ind_obj)
+            except Exception:
+                if g_exceptions_verbose:    exception_info()
+                message("skipping bad link / annot item %i.\n" % i)
+
+    def _addWidget(self, field_type, field_name):
+        page = self._pdf_page()
+        pdf = page.doc()
+        annot = JM_create_widget(pdf, page, field_type, field_name)
+        if not annot.m_internal:
+            raise RuntimeError( "cannot create widget")
+        JM_add_annot_id(annot, "W")
+        return Annot(annot)
+
+    def _apply_redactions(self, text, images, graphics):
+        page = self._pdf_page()
+        opts = mupdf.PdfRedactOptions()
+        opts.black_boxes = 0  # no black boxes
+        opts.text = text  # how to treat text
+        opts.image_method = images  # how to treat images
+        opts.line_art = graphics  # how to treat vector graphics
+        success = mupdf.pdf_redact_page(page.doc(), page, opts)
+        return success
+
+    def _erase(self):
+        self._reset_annot_refs()
+        try:
+            self.parent._forget_page(self)
+        except Exception:
+            exception_info()
+            pass
+        self.parent = None
+        self.thisown = False
+        self.number = None
+        self.this = None
+
+    def _count_q_balance(self):
+        """Count missing graphic state pushs and pops.
+
+        Returns:
+            A pair of integers (push, pop). Push is the number of missing
+            PDF "q" commands, pop is the number of "Q" commands.
+            A balanced graphics state for the page will be reached if its
+            /Contents is prepended with 'push' copies of string "q\n"
+            and appended with 'pop' copies of "\nQ".
+        """
+        page = _as_pdf_page(self)  # need the underlying PDF page
+        res = mupdf.pdf_dict_get(  # access /Resources
+            page.obj(),
+            mupdf.PDF_ENUM_NAME_Resources,
+        )
+        cont = mupdf.pdf_dict_get(  # access /Contents
+            page.obj(),
+            mupdf.PDF_ENUM_NAME_Contents,
+        )
+        pdf = _as_pdf_document(self.parent)  # need underlying PDF document
+
+        # return value of MuPDF function
+        return mupdf.pdf_count_q_balance_outparams_fn(pdf, res, cont)
+
+    def _get_optional_content(self, oc: OptInt) -> OptStr:
+        if oc is None or oc == 0:
+            return None
+        doc = self.parent
+        check = doc.xref_object(oc, compressed=True)
+        if not ("/Type/OCG" in check or "/Type/OCMD" in check):
+            #log( 'raising "bad optional content"')
+            raise ValueError("bad optional content: 'oc'")
+        #log( 'Looking at self._get_resource_properties()')
+        props = {}
+        for p, x in self._get_resource_properties():
+            props[x] = p
+        if oc in props.keys():
+            return props[oc]
+        i = 0
+        mc = "MC%i" % i
+        while mc in props.values():
+            i += 1
+            mc = "MC%i" % i
+        self._set_resource_property(mc, oc)
+        #log( 'returning {mc=}')
+        return mc
+
+    def _get_resource_properties(self):
+        '''
+        page list Resource/Properties
+        '''
+        page = self._pdf_page()
+        rc = JM_get_resource_properties(page.obj())
+        return rc
+
+    def _get_textpage(self, clip=None, flags=0, matrix=None):
+        if g_use_extra:
+            ll_tpage = extra.page_get_textpage(self.this, clip, flags, matrix)
+            tpage = mupdf.FzStextPage(ll_tpage)
+            return tpage
+        page = self.this
+        options = mupdf.FzStextOptions(flags)
+        rect = JM_rect_from_py(clip)
+        # Default to page's rect if `clip` not specified, for #2048.
+        rect = mupdf.fz_bound_page(page) if clip is None else JM_rect_from_py(clip)
+        ctm = JM_matrix_from_py(matrix)
+        tpage = mupdf.FzStextPage(rect)
+        dev = mupdf.fz_new_stext_device(tpage, options)
+        if _globals.no_device_caching:
+            mupdf.fz_enable_device_hints( dev, mupdf.FZ_NO_CACHE)
+        if isinstance(page, mupdf.FzPage):
+            pass
+        elif isinstance(page, mupdf.PdfPage):
+            page = page.super()
+        else:
+            assert 0, f'Unrecognised {type(page)=}'
+        mupdf.fz_run_page(page, dev, ctm, mupdf.FzCookie())
+        mupdf.fz_close_device(dev)
+        return tpage
+
+    def _insert_image(self,
+            filename=None, pixmap=None, stream=None, imask=None, clip=None,
+            overlay=1, rotate=0, keep_proportion=1, oc=0, width=0, height=0,
+            xref=0, alpha=-1, _imgname=None, digests=None
+            ):
+        maskbuf = mupdf.FzBuffer()
+        page = self._pdf_page()
+        # This will create an empty PdfDocument with a call to
+        # pdf_new_document() then assign page.doc()'s return value to it (which
+        # drop the original empty pdf_document).
+        pdf = page.doc()
+        w = width
+        h = height
+        img_xref = xref
+        rc_digest = 0
+
+        do_process_pixmap = 1
+        do_process_stream = 1
+        do_have_imask = 1
+        do_have_image = 1
+        do_have_xref = 1
+
+        if xref > 0:
+            ref = mupdf.pdf_new_indirect(pdf, xref, 0)
+            w = mupdf.pdf_to_int( mupdf.pdf_dict_geta( ref, PDF_NAME('Width'), PDF_NAME('W')))
+            h = mupdf.pdf_to_int( mupdf.pdf_dict_geta( ref, PDF_NAME('Height'), PDF_NAME('H')))
+            if w + h == 0:
+                raise ValueError( MSG_IS_NO_IMAGE)
+            #goto have_xref()
+            do_process_pixmap = 0
+            do_process_stream = 0
+            do_have_imask = 0
+            do_have_image = 0
+
+        else:
+            if stream:
+                imgbuf = JM_BufferFromBytes(stream)
+                do_process_pixmap = 0
+            else:
+                if filename:
+                    imgbuf = mupdf.fz_read_file(filename)
+                    #goto have_stream()
+                    do_process_pixmap = 0
+
+        if do_process_pixmap:
+            #log( 'do_process_pixmap')
+            # process pixmap ---------------------------------
+            arg_pix = pixmap.this
+            w = arg_pix.w()
+            h = arg_pix.h()
+            digest = mupdf.fz_md5_pixmap2(arg_pix)
+            md5_py = digest
+            temp = digests.get(md5_py, None)
+            if temp is not None:
+                img_xref = temp
+                ref = mupdf.pdf_new_indirect(page.doc(), img_xref, 0)
+                #goto have_xref()
+                do_process_stream = 0
+                do_have_imask = 0
+                do_have_image = 0
+            else:
+                if arg_pix.alpha() == 0:
+                    image = mupdf.fz_new_image_from_pixmap(arg_pix, mupdf.FzImage())
+                else:
+                    pm = mupdf.fz_convert_pixmap(
+                            arg_pix,
+                            mupdf.FzColorspace(),
+                            mupdf.FzColorspace(),
+                            mupdf.FzDefaultColorspaces(None),
+                            mupdf.FzColorParams(),
+                            1,
+                            )
+                    pm.alpha = 0
+                    pm.colorspace = None
+                    mask = mupdf.fz_new_image_from_pixmap(pm, mupdf.FzImage())
+                    image = mupdf.fz_new_image_from_pixmap(arg_pix, mask)
+                #goto have_image()
+                do_process_stream = 0
+                do_have_imask = 0
+
+        if do_process_stream:
+            #log( 'do_process_stream')
+            # process stream ---------------------------------
+            state = mupdf.FzMd5()
+            if mupdf_cppyy:
+                mupdf.fz_md5_update_buffer( state, imgbuf)
+            else:
+                mupdf.fz_md5_update(state, imgbuf.m_internal.data, imgbuf.m_internal.len)
+            if imask:
+                maskbuf = JM_BufferFromBytes(imask)
+                if mupdf_cppyy:
+                    mupdf.fz_md5_update_buffer( state, maskbuf)
+                else:
+                    mupdf.fz_md5_update(state, maskbuf.m_internal.data, maskbuf.m_internal.len)
+            digest = mupdf.fz_md5_final2(state)
+            md5_py = bytes(digest)
+            temp = digests.get(md5_py, None)
+            if temp is not None:
+                img_xref = temp
+                ref = mupdf.pdf_new_indirect(page.doc(), img_xref, 0)
+                w = mupdf.pdf_to_int( mupdf.pdf_dict_geta( ref, PDF_NAME('Width'), PDF_NAME('W')))
+                h = mupdf.pdf_to_int( mupdf.pdf_dict_geta( ref, PDF_NAME('Height'), PDF_NAME('H')))
+                #goto have_xref()
+                do_have_imask = 0
+                do_have_image = 0
+            else:
+                image = mupdf.fz_new_image_from_buffer(imgbuf)
+                w = image.w()
+                h = image.h()
+                if not imask:
+                    #goto have_image()
+                    do_have_imask = 0
+
+        if do_have_imask:
+            # `fz_compressed_buffer` is reference counted and
+            # `mupdf.fz_new_image_from_compressed_buffer2()`
+            # is povided as a Swig-friendly wrapper for
+            # `fz_new_image_from_compressed_buffer()`, so we can do things
+            # straightfowardly.
+            #
+            cbuf1 = mupdf.fz_compressed_image_buffer( image)
+            if not cbuf1.m_internal:
+                raise ValueError( "uncompressed image cannot have mask")
+            bpc = image.bpc()
+            colorspace = image.colorspace()
+            xres, yres = mupdf.fz_image_resolution(image)
+            mask = mupdf.fz_new_image_from_buffer(maskbuf)
+            image = mupdf.fz_new_image_from_compressed_buffer2(
+                    w,
+                    h,
+                    bpc,
+                    colorspace,
+                    xres,
+                    yres,
+                    1,  # interpolate
+                    0,  # imagemask,
+                    list(), # decode
+                    list(), # colorkey
+                    cbuf1,
+                    mask,
+                    )
+            
+        if do_have_image:
+            #log( 'do_have_image')
+            ref = mupdf.pdf_add_image(pdf, image)
+            if oc:
+                JM_add_oc_object(pdf, ref, oc)
+            img_xref = mupdf.pdf_to_num(ref)
+            digests[md5_py] = img_xref
+            rc_digest = 1
+
+        if do_have_xref:
+            #log( 'do_have_xref')
+            resources = mupdf.pdf_dict_get_inheritable(page.obj(), PDF_NAME('Resources'))
+            if not resources.m_internal:
+                resources = mupdf.pdf_dict_put_dict(page.obj(), PDF_NAME('Resources'), 2)
+            xobject = mupdf.pdf_dict_get(resources, PDF_NAME('XObject'))
+            if not xobject.m_internal:
+                xobject = mupdf.pdf_dict_put_dict(resources, PDF_NAME('XObject'), 2)
+            mat = calc_image_matrix(w, h, clip, rotate, keep_proportion)
+            mupdf.pdf_dict_puts(xobject, _imgname, ref)
+            nres = mupdf.fz_new_buffer(50)
+            s = f"\nq\n{_format_g((mat.a, mat.b, mat.c, mat.d, mat.e, mat.f))} cm\n/{_imgname} Do\nQ\n"
+            #s = s.replace('\n', '\r\n')
+            mupdf.fz_append_string(nres, s)
+            JM_insert_contents(pdf, page.obj(), nres, overlay)
+
+        if rc_digest:
+            return img_xref, digests
+        else:
+            return img_xref, None
+
+    def _insertFont(self, fontname, bfname, fontfile, fontbuffer, set_simple, idx, wmode, serif, encoding, ordering):
+        page = self._pdf_page()
+        pdf = page.doc()
+
+        value = JM_insert_font(pdf, bfname, fontfile,fontbuffer, set_simple, idx, wmode, serif, encoding, ordering)
+        # get the objects /Resources, /Resources/Font
+        resources = mupdf.pdf_dict_get_inheritable(page.obj(), PDF_NAME('Resources'))
+        if not resources.pdf_is_dict():
+            resources = mupdf.pdf_dict_put_dict(page.obj(), PDF_NAME("Resources"), 5)
+        fonts = mupdf.pdf_dict_get(resources, PDF_NAME('Font'))
+        if not fonts.m_internal:    # page has no fonts yet
+            fonts = mupdf.pdf_new_dict(pdf, 5)
+            mupdf.pdf_dict_putl(page.obj(), fonts, PDF_NAME('Resources'), PDF_NAME('Font'))
+        # store font in resources and fonts objects will contain named reference to font
+        _, xref = JM_INT_ITEM(value, 0)
+        if not xref:
+            raise RuntimeError( "cannot insert font")
+        font_obj = mupdf.pdf_new_indirect(pdf, xref, 0)
+        mupdf.pdf_dict_puts(fonts, fontname, font_obj)
+        return value
+
+    def _load_annot(self, name, xref):
+        page = self._pdf_page()
+        if xref == 0:
+            annot = JM_get_annot_by_name(page, name)
+        else:
+            annot = JM_get_annot_by_xref(page, xref)
+        if annot.m_internal:
+            return Annot(annot)
+
+    def _makePixmap(self, doc, ctm, cs, alpha=0, annots=1, clip=None):
+        pix = JM_pixmap_from_page(doc, self.this, ctm, cs, alpha, annots, clip)
+        return Pixmap(pix)
+
+    def _other_box(self, boxtype):
+        rect = mupdf.FzRect( mupdf.FzRect.Fixed_INFINITE)
+        page = _as_pdf_page(self.this, required=False)
+        if page.m_internal:
+            obj = mupdf.pdf_dict_gets( page.obj(), boxtype)
+            if mupdf.pdf_is_array(obj):
+                rect = mupdf.pdf_to_rect(obj)
+        if mupdf.fz_is_infinite_rect( rect):
+            return
+        return JM_py_from_rect(rect)
+
+    def _pdf_page(self, required=True):
+        return _as_pdf_page(self.this, required=required)
+
+    def _reset_annot_refs(self):
+        """Invalidate / delete all annots of this page."""
+        self._annot_refs.clear()
+
+    def _set_opacity(self, gstate=None, CA=1, ca=1, blendmode=None):
+
+        if CA >= 1 and ca >= 1 and blendmode is None:
+            return
+        tCA = int(round(max(CA , 0) * 100))
+        if tCA >= 100:
+            tCA = 99
+        tca = int(round(max(ca, 0) * 100))
+        if tca >= 100:
+            tca = 99
+        gstate = "fitzca%02i%02i" % (tCA, tca)
+
+        if not gstate:
+            return
+        page = _as_pdf_page(self.this)
+        resources = mupdf.pdf_dict_get(page.obj(), PDF_NAME('Resources'))
+        if not resources.m_internal:
+            resources = mupdf.pdf_dict_put_dict(page.obj(), PDF_NAME('Resources'), 2)
+        extg = mupdf.pdf_dict_get(resources, PDF_NAME('ExtGState'))
+        if not extg.m_internal:
+            extg = mupdf.pdf_dict_put_dict(resources, PDF_NAME('ExtGState'), 2)
+        n = mupdf.pdf_dict_len(extg)
+        for i in range(n):
+            o1 = mupdf.pdf_dict_get_key(extg, i)
+            name = mupdf.pdf_to_name(o1)
+            if name == gstate:
+                return gstate
+        opa = mupdf.pdf_new_dict(page.doc(), 3)
+        mupdf.pdf_dict_put_real(opa, PDF_NAME('CA'), CA)
+        mupdf.pdf_dict_put_real(opa, PDF_NAME('ca'), ca)
+        mupdf.pdf_dict_puts(extg, gstate, opa)
+        return gstate
+
+    def _set_pagebox(self, boxtype, rect):
+        doc = self.parent
+        if doc is None:
+            raise ValueError("orphaned object: parent is None")
+
+        if not doc.is_pdf:
+            raise ValueError("is no PDF")
+
+        valid_boxes = ("CropBox", "BleedBox", "TrimBox", "ArtBox")
+
+        if boxtype not in valid_boxes:
+            raise ValueError("bad boxtype")
+
+        rect = Rect(rect)
+        mb = self.mediabox
+        rect = Rect(rect[0], mb.y1 - rect[3], rect[2], mb.y1 - rect[1])
+        if not (mb.x0 <= rect.x0 < rect.x1 <= mb.x1 and mb.y0 <= rect.y0 < rect.y1 <= mb.y1):
+            raise ValueError(f"{boxtype} not in MediaBox")
+
+        doc.xref_set_key(self.xref, boxtype, f"[{_format_g(tuple(rect))}]")
+
+    def _set_resource_property(self, name, xref):
+        page = self._pdf_page()
+        JM_set_resource_property(page.obj(), name, xref)
+
+    def _show_pdf_page(self, fz_srcpage, overlay=1, matrix=None, xref=0, oc=0, clip=None, graftmap=None, _imgname=None):
+        cropbox = JM_rect_from_py(clip)
+        mat = JM_matrix_from_py(matrix)
+        rc_xref = xref
+        tpage = _as_pdf_page(self.this)
+        tpageref = tpage.obj()
+        pdfout = tpage.doc()    # target PDF
+        ENSURE_OPERATION(pdfout)
+        #-------------------------------------------------------------
+        # convert the source page to a Form XObject
+        #-------------------------------------------------------------
+        xobj1 = JM_xobject_from_page(pdfout, fz_srcpage, xref, graftmap.this)
+        if not rc_xref:
+            rc_xref = mupdf.pdf_to_num(xobj1)
+
+        #-------------------------------------------------------------
+        # create referencing XObject (controls display on target page)
+        #-------------------------------------------------------------
+        # fill reference to xobj1 into the /Resources
+        #-------------------------------------------------------------
+        subres1 = mupdf.pdf_new_dict(pdfout, 5)
+        mupdf.pdf_dict_puts(subres1, "fullpage", xobj1)
+        subres = mupdf.pdf_new_dict(pdfout, 5)
+        mupdf.pdf_dict_put(subres, PDF_NAME('XObject'), subres1)
+
+        res = mupdf.fz_new_buffer(20)
+        mupdf.fz_append_string(res, "/fullpage Do")
+
+        xobj2 = mupdf.pdf_new_xobject(pdfout, cropbox, mat, subres, res)
+        if oc > 0:
+            JM_add_oc_object(pdfout, mupdf.pdf_resolve_indirect(xobj2), oc)
+
+        #-------------------------------------------------------------
+        # update target page with xobj2:
+        #-------------------------------------------------------------
+        # 1. insert Xobject in Resources
+        #-------------------------------------------------------------
+        resources = mupdf.pdf_dict_get_inheritable(tpageref, PDF_NAME('Resources'))
+        if not resources.m_internal:
+            resources = mupdf.pdf_dict_put_dict(tpageref,PDF_NAME('Resources'), 5)
+        subres = mupdf.pdf_dict_get(resources, PDF_NAME('XObject'))
+        if not subres.m_internal:
+            subres = mupdf.pdf_dict_put_dict(resources, PDF_NAME('XObject'), 5)
+
+        mupdf.pdf_dict_puts(subres, _imgname, xobj2)
+
+        #-------------------------------------------------------------
+        # 2. make and insert new Contents object
+        #-------------------------------------------------------------
+        nres = mupdf.fz_new_buffer(50) # buffer for Do-command
+        mupdf.fz_append_string(nres, " q /")   # Do-command
+        mupdf.fz_append_string(nres, _imgname)
+        mupdf.fz_append_string(nres, " Do Q ")
+
+        JM_insert_contents(pdfout, tpageref, nres, overlay)
+        return rc_xref
+
+    def add_caret_annot(self, point: point_like) -> Annot:
+        """Add a 'Caret' annotation."""
+        old_rotation = annot_preprocess(self)
+        try:
+            annot = self._add_caret_annot(point)
+        finally:
+            if old_rotation != 0:
+                self.set_rotation(old_rotation)
+        annot = Annot( annot)
+        annot_postprocess(self, annot)
+        assert hasattr( annot, 'parent')
+        return annot
+
+    def add_circle_annot(self, rect: rect_like) -> Annot:
+        """Add a 'Circle' (ellipse, oval) annotation."""
+        old_rotation = annot_preprocess(self)
+        try:
+            annot = self._add_square_or_circle(rect, mupdf.PDF_ANNOT_CIRCLE)
+        finally:
+            if old_rotation != 0:
+                self.set_rotation(old_rotation)
+        annot_postprocess(self, annot)
+        return annot
+
+    def add_file_annot(
+            self,
+            point: point_like,
+            buffer_: ByteString,
+            filename: str,
+            ufilename: OptStr =None,
+            desc: OptStr =None,
+            icon: OptStr =None
+            ) -> Annot:
+        """Add a 'FileAttachment' annotation."""
+        old_rotation = annot_preprocess(self)
+        try:
+            annot = self._add_file_annot(point,
+                    buffer_,
+                    filename,
+                    ufilename=ufilename,
+                    desc=desc,
+                    icon=icon,
+                    )
+        finally:
+            if old_rotation != 0:
+                self.set_rotation(old_rotation)
+        annot_postprocess(self, annot)
+        return annot
+
+    def add_freetext_annot(
+            self,
+            rect: rect_like,
+            text: str,
+            *,
+            fontsize: float =11,
+            fontname: OptStr =None,
+            text_color: OptSeq =None,
+            fill_color: OptSeq =None,
+            border_color: OptSeq =None,
+            border_width: float =0,
+            dashes: OptSeq =None,
+            callout: OptSeq =None,
+            line_end: int=mupdf.PDF_ANNOT_LE_OPEN_ARROW,
+            opacity: float =1,
+            align: int =0,
+            rotate: int =0,
+            richtext=False,
+            style=None,
+            ) -> Annot:
+        """Add a 'FreeText' annotation."""
+
+        old_rotation = annot_preprocess(self)
+        try:
+            annot = self._add_freetext_annot(
+                    rect,
+                    text,
+                    fontsize=fontsize,
+                    fontname=fontname,
+                    text_color=text_color,
+                    fill_color=fill_color,
+                    border_color=border_color,
+                    border_width=border_width,
+                    dashes=dashes,
+                    callout=callout,
+                    line_end=line_end,
+                    opacity=opacity,
+                    align=align,
+                    rotate=rotate,
+                    richtext=richtext,
+                    style=style,
+                    )
+        finally:
+            if old_rotation != 0:
+                self.set_rotation(old_rotation)
+        annot_postprocess(self, annot)
+        return annot
+
+    def add_highlight_annot(self, quads=None, start=None,
+                          stop=None, clip=None) -> Annot:
+        """Add a 'Highlight' annotation."""
+        if quads is None:
+            q = get_highlight_selection(self, start=start, stop=stop, clip=clip)
+        else:
+            q = CheckMarkerArg(quads)
+        ret = self._add_text_marker(q, mupdf.PDF_ANNOT_HIGHLIGHT)
+        return ret
+
+    def add_ink_annot(self, handwriting: list) -> Annot:
+        """Add a 'Ink' ('handwriting') annotation.
+
+        The argument must be a list of lists of point_likes.
+        """
+        old_rotation = annot_preprocess(self)
+        try:
+            annot = self._add_ink_annot(handwriting)
+        finally:
+            if old_rotation != 0:
+                self.set_rotation(old_rotation)
+        annot_postprocess(self, annot)
+        return annot
+
+    def add_line_annot(self, p1: point_like, p2: point_like) -> Annot:
+        """Add a 'Line' annotation."""
+        old_rotation = annot_preprocess(self)
+        try:
+            annot = self._add_line_annot(p1, p2)
+        finally:
+            if old_rotation != 0:
+                self.set_rotation(old_rotation)
+        annot_postprocess(self, annot)
+        return annot
+
+    def add_polygon_annot(self, points: list) -> Annot:
+        """Add a 'Polygon' annotation."""
+        old_rotation = annot_preprocess(self)
+        try:
+            annot = self._add_multiline(points, mupdf.PDF_ANNOT_POLYGON)
+        finally:
+            if old_rotation != 0:
+                self.set_rotation(old_rotation)
+        annot_postprocess(self, annot)
+        return annot
+
+    def add_polyline_annot(self, points: list) -> Annot:
+        """Add a 'PolyLine' annotation."""
+        old_rotation = annot_preprocess(self)
+        try:
+            annot = self._add_multiline(points, mupdf.PDF_ANNOT_POLY_LINE)
+        finally:
+            if old_rotation != 0:
+                self.set_rotation(old_rotation)
+        annot_postprocess(self, annot)
+        return annot
+
+    def add_rect_annot(self, rect: rect_like) -> Annot:
+        """Add a 'Square' (rectangle) annotation."""
+        old_rotation = annot_preprocess(self)
+        try:
+            annot = self._add_square_or_circle(rect, mupdf.PDF_ANNOT_SQUARE)
+        finally:
+            if old_rotation != 0:
+                self.set_rotation(old_rotation)
+        annot_postprocess(self, annot)
+        return annot
+
+    def add_redact_annot(
+            self,
+            quad,
+            text: OptStr =None,
+            fontname: OptStr =None,
+            fontsize: float =11,
+            align: int =0,
+            fill: OptSeq =None,
+            text_color: OptSeq =None,
+            cross_out: bool =True,
+            ) -> Annot:
+        """Add a 'Redact' annotation."""
+        da_str = None
+        if text and not set(string.whitespace).issuperset(text):
+            CheckColor(fill)
+            CheckColor(text_color)
+            if not fontname:
+                fontname = "Helv"
+            if not fontsize:
+                fontsize = 11
+            if not text_color:
+                text_color = (0, 0, 0)
+            if hasattr(text_color, "__float__"):
+                text_color = (text_color, text_color, text_color)
+            if len(text_color) > 3:
+                text_color = text_color[:3]
+            fmt = "{:g} {:g} {:g} rg /{f:s} {s:g} Tf"
+            da_str = fmt.format(*text_color, f=fontname, s=fontsize)
+            if fill is None:
+                fill = (1, 1, 1)
+            if fill:
+                if hasattr(fill, "__float__"):
+                    fill = (fill, fill, fill)
+                if len(fill) > 3:
+                    fill = fill[:3]
+        else:
+            text = None
+
+        old_rotation = annot_preprocess(self)
+        try:
+            annot = self._add_redact_annot(quad, text=text, da_str=da_str,
+                       align=align, fill=fill)
+        finally:
+            if old_rotation != 0:
+                self.set_rotation(old_rotation)
+        annot_postprocess(self, annot)
+        #-------------------------------------------------------------
+        # change appearance to show a crossed-out rectangle
+        #-------------------------------------------------------------
+        if cross_out:
+            ap_tab = annot._getAP().splitlines()[:-1]  # get the 4 commands only
+            _, LL, LR, UR, UL = ap_tab
+            ap_tab.append(LR)
+            ap_tab.append(LL)
+            ap_tab.append(UR)
+            ap_tab.append(LL)
+            ap_tab.append(UL)
+            ap_tab.append(b"S")
+            ap = b"\n".join(ap_tab)
+            annot._setAP(ap, 0)
+        return annot
+
+    def add_squiggly_annot(
+            self,
+            quads=None,
+            start=None,
+            stop=None,
+            clip=None,
+            ) -> Annot:
+        """Add a 'Squiggly' annotation."""
+        if quads is None:
+            q = get_highlight_selection(self, start=start, stop=stop, clip=clip)
+        else:
+            q = CheckMarkerArg(quads)
+        return self._add_text_marker(q, mupdf.PDF_ANNOT_SQUIGGLY)
+
+    def add_stamp_annot(self, rect: rect_like, stamp=0) -> Annot:
+        """Add a ('rubber') 'Stamp' annotation."""
+        old_rotation = annot_preprocess(self)
+        try:
+            annot = self._add_stamp_annot(rect, stamp)
+        finally:
+            if old_rotation != 0:
+                self.set_rotation(old_rotation)
+        annot_postprocess(self, annot)
+        return annot
+
+    def add_strikeout_annot(self, quads=None, start=None, stop=None, clip=None) -> Annot:
+        """Add a 'StrikeOut' annotation."""
+        if quads is None:
+            q = get_highlight_selection(self, start=start, stop=stop, clip=clip)
+        else:
+            q = CheckMarkerArg(quads)
+        return self._add_text_marker(q, mupdf.PDF_ANNOT_STRIKE_OUT)
+
+    def add_text_annot(self, point: point_like, text: str, icon: str ="Note") -> Annot:
+        """Add a 'Text' (sticky note) annotation."""
+        old_rotation = annot_preprocess(self)
+        try:
+            annot = self._add_text_annot(point, text, icon=icon)
+        finally:
+            if old_rotation != 0:
+                self.set_rotation(old_rotation)
+        annot_postprocess(self, annot)
+        return annot
+
+    def add_underline_annot(self, quads=None, start=None, stop=None, clip=None) -> Annot:
+        """Add a 'Underline' annotation."""
+        if quads is None:
+            q = get_highlight_selection(self, start=start, stop=stop, clip=clip)
+        else:
+            q = CheckMarkerArg(quads)
+        return self._add_text_marker(q, mupdf.PDF_ANNOT_UNDERLINE)
+
+    def add_widget(self, widget: Widget) -> Annot:
+        """Add a 'Widget' (form field)."""
+        CheckParent(self)
+        doc = self.parent
+        if not doc.is_pdf:
+            raise ValueError("is no PDF")
+        widget._validate()
+        annot = self._addWidget(widget.field_type, widget.field_name)
+        if not annot:
+            return None
+        annot.thisown = True
+        annot.parent = weakref.proxy(self) # owning page object
+        self._annot_refs[id(annot)] = annot
+        widget.parent = annot.parent
+        widget._annot = annot
+        widget.update()
+        return annot
+
+    def annot_names(self):
+        '''
+        page get list of annot names
+        '''
+        """List of names of annotations, fields and links."""
+        CheckParent(self)
+        page = self._pdf_page(required=False)
+        if not page.m_internal:
+            return []
+        return JM_get_annot_id_list(page)
+
+    def annot_xrefs(self):
+        '''
+        List of xref numbers of annotations, fields and links.
+        '''
+        return JM_get_annot_xref_list2(self)
+    
+    def annots(self, types=None):
+        """ Generator over the annotations of a page.
+
+        Args:
+            types: (list) annotation types to subselect from. If none,
+                   all annotations are returned. E.g. types=[PDF_ANNOT_LINE]
+                   will only yield line annotations.
+        """
+        skip_types = (mupdf.PDF_ANNOT_LINK, mupdf.PDF_ANNOT_POPUP, mupdf.PDF_ANNOT_WIDGET)
+        if not hasattr(types, "__getitem__"):
+            annot_xrefs = [a[0] for a in self.annot_xrefs() if a[1] not in skip_types]
+        else:
+            annot_xrefs = [a[0] for a in self.annot_xrefs() if a[1] in types and a[1] not in skip_types]
+        for xref in annot_xrefs:
+            annot = self.load_annot(xref)
+            annot._yielded=True
+            yield annot
+
+    def recolor(self, components=1):
+        """Convert colorspaces of objects on the page.
+        
+        Valid values are 1, 3 and 4.
+        """
+        if components not in (1, 3, 4):
+            raise ValueError("components must be one of 1, 3, 4")
+        pdfdoc = _as_pdf_document(self.parent)
+        ropt = mupdf.pdf_recolor_options()
+        ropt.num_comp = components
+        ropts = mupdf.PdfRecolorOptions(ropt)
+        mupdf.pdf_recolor_page(pdfdoc, self.number, ropts)
+
+    @property
+    def artbox(self):
+        """The ArtBox"""
+        rect = self._other_box("ArtBox")
+        if rect is None:
+            return self.cropbox
+        mb = self.mediabox
+        return Rect(rect[0], mb.y1 - rect[3], rect[2], mb.y1 - rect[1])
+
+    @property
+    def bleedbox(self):
+        """The BleedBox"""
+        rect = self._other_box("BleedBox")
+        if rect is None:
+            return self.cropbox
+        mb = self.mediabox
+        return Rect(rect[0], mb.y1 - rect[3], rect[2], mb.y1 - rect[1])
+
+    def bound(self):
+        """Get page rectangle."""
+        CheckParent(self)
+        page = _as_fz_page(self.this)
+        val = mupdf.fz_bound_page(page)
+        val = Rect(val)
+        
+        if val.is_infinite and self.parent.is_pdf:
+            cb = self.cropbox
+            w, h = cb.width, cb.height
+            if self.rotation not in (0, 180):
+                w, h = h, w
+            val = Rect(0, 0, w, h)
+            msg = TOOLS.mupdf_warnings(reset=False).splitlines()[-1]
+            message(msg)
+        
+        return val
+
+    def clean_contents(self, sanitize=1):
+        if not sanitize and not self.is_wrapped:
+            self.wrap_contents()
+        page = _as_pdf_page( self.this, required=False)
+        if not page.m_internal:
+            return
+        filter_ = _make_PdfFilterOptions(recurse=1, sanitize=sanitize)
+        mupdf.pdf_filter_page_contents( page.doc(), page, filter_)
+    
+    @property
+    def cropbox(self):
+        """The CropBox."""
+        CheckParent(self)
+        page = self._pdf_page(required=False)
+        if not page.m_internal:
+            val = mupdf.fz_bound_page(self.this)
+        else:
+            val = JM_cropbox(page.obj())
+        val = Rect(val)
+
+        return val
+
+    @property
+    def cropbox_position(self):
+        return self.cropbox.tl
+
+    def delete_annot(self, annot):
+        """Delete annot and return next one."""
+        CheckParent(self)
+        CheckParent(annot)
+
+        page = self._pdf_page()
+        while 1:
+            # first loop through all /IRT annots and remove them
+            irt_annot = JM_find_annot_irt(annot.this)
+            if not irt_annot:    # no more there
+                break
+            mupdf.pdf_delete_annot(page, irt_annot.this)
+        nextannot = mupdf.pdf_next_annot(annot.this)   # store next
+        mupdf.pdf_delete_annot(page, annot.this)
+        val = Annot(nextannot)
+
+        if val:
+            val.thisown = True
+            val.parent = weakref.proxy(self) # owning page object
+            val.parent._annot_refs[id(val)] = val
+        annot._erase()
+        return val
+
+    def delete_link(self, linkdict):
+        """Delete a Link."""
+        CheckParent(self)
+        if not isinstance( linkdict, dict):
+            return  # have no dictionary
+
+        def finished():
+            if linkdict["xref"] == 0: return
+            try:
+                linkid = linkdict["id"]
+                linkobj = self._annot_refs[linkid]
+                linkobj._erase()
+            except Exception:
+                # Don't print this exception, to match classic. Issue #2841.
+                if g_exceptions_verbose > 1:    exception_info()
+                pass
+
+        page = _as_pdf_page(self.this, required=False)
+        if not page.m_internal:
+            return finished()   # have no PDF
+        xref = linkdict[dictkey_xref]
+        if xref < 1:
+            return finished()   # invalid xref
+        annots = mupdf.pdf_dict_get( page.obj(), PDF_NAME('Annots'))
+        if not annots.m_internal:
+            return finished()   # have no annotations
+        len_ = mupdf.pdf_array_len( annots)
+        if len_ == 0:
+            return finished()
+        oxref = 0
+        for i in range( len_):
+            oxref = mupdf.pdf_to_num( mupdf.pdf_array_get( annots, i))
+            if xref == oxref:
+                break   # found xref in annotations
+
+        if xref != oxref:
+            return finished()   # xref not in annotations
+        mupdf.pdf_array_delete( annots, i) # delete entry in annotations
+        mupdf.pdf_delete_object( page.doc(), xref) # delete link object
+        mupdf.pdf_dict_put( page.obj(), PDF_NAME('Annots'), annots)
+        JM_refresh_links( page)
+
+        return finished()
+
+    @property
+    def derotation_matrix(self) -> Matrix:
+        """Reflects page de-rotation."""
+        if g_use_extra:
+            return Matrix(extra.Page_derotate_matrix( self.this))
+        pdfpage = self._pdf_page(required=False)
+        if not pdfpage.m_internal:
+            return Matrix(mupdf.FzRect(mupdf.FzRect.UNIT))
+        return Matrix(JM_derotate_page_matrix(pdfpage))
+
+    def extend_textpage(self, tpage, flags=0, matrix=None):
+        page = self.this
+        tp = tpage.this
+        assert isinstance( tp, mupdf.FzStextPage)
+        options = mupdf.FzStextOptions()
+        options.flags = flags
+        ctm = JM_matrix_from_py(matrix)
+        dev = mupdf.FzDevice(tp, options)
+        mupdf.fz_run_page( page, dev, ctm, mupdf.FzCookie())
+        mupdf.fz_close_device( dev)
+
+    @property
+    def first_annot(self):
+        """First annotation."""
+        CheckParent(self)
+        page = self._pdf_page(required=False)
+        if not page.m_internal:
+            return
+        annot = mupdf.pdf_first_annot(page)
+        if not annot.m_internal:
+            return
+        val = Annot(annot)
+        val.thisown = True
+        val.parent = weakref.proxy(self) # owning page object
+        self._annot_refs[id(val)] = val
+        return val
+
+    @property
+    def first_link(self):
+        '''
+        First link on page
+        '''
+        return self.load_links()
+
+    @property
+    def first_widget(self):
+        """First widget/field."""
+        CheckParent(self)
+        annot = 0
+        page = self._pdf_page(required=False)
+        if not page.m_internal:
+            return
+        annot = mupdf.pdf_first_widget(page)
+        if not annot.m_internal:
+            return
+        val = Annot(annot)
+        val.thisown = True
+        val.parent = weakref.proxy(self) # owning page object
+        self._annot_refs[id(val)] = val
+        widget = Widget()
+        TOOLS._fill_widget(val, widget)
+        val = widget
+        return val
+
+    def get_bboxlog(self, layers=None):
+        CheckParent(self)
+        old_rotation = self.rotation
+        if old_rotation != 0:
+            self.set_rotation(0)
+        page = self.this
+        rc = []
+        inc_layers = True if layers else False
+        dev = JM_new_bbox_device( rc, inc_layers)
+        mupdf.fz_run_page( page, dev, mupdf.FzMatrix(), mupdf.FzCookie())
+        mupdf.fz_close_device( dev)
+
+        if old_rotation != 0:
+            self.set_rotation(old_rotation)
+        return rc
+
+    def get_cdrawings(self, extended=None, callback=None, method=None):
+        """Extract vector graphics ("line art") from the page."""
+        CheckParent(self)
+        old_rotation = self.rotation
+        if old_rotation != 0:
+            self.set_rotation(0)
+        page = self.this
+        if isinstance(page, mupdf.PdfPage):
+            # Downcast pdf_page to fz_page.
+            page = mupdf.FzPage(page)
+        assert isinstance(page, mupdf.FzPage), f'{self.this=}'
+        clips = True if extended else False
+        prect = mupdf.fz_bound_page(page)
+        if g_use_extra:
+            rc = extra.get_cdrawings(page, extended, callback, method)
+        else:
+            rc = list()
+            if callable(callback) or method is not None:
+                dev = JM_new_lineart_device_Device(callback, clips, method)
+            else:
+                dev = JM_new_lineart_device_Device(rc, clips, method)
+            dev.ptm = mupdf.FzMatrix(1, 0, 0, -1, 0, prect.y1)
+            mupdf.fz_run_page(page, dev, mupdf.FzMatrix(), mupdf.FzCookie())
+            mupdf.fz_close_device(dev)
+
+        if old_rotation != 0:
+            self.set_rotation(old_rotation)
+        if callable(callback) or method is not None:
+            return
+        return rc
+
+    def get_contents(self):
+        """Get xrefs of /Contents objects."""
+        CheckParent(self)
+        ret = []
+        page = _as_pdf_page(self.this)
+        obj = page.obj()
+        contents = mupdf.pdf_dict_get(obj, mupdf.PDF_ENUM_NAME_Contents)
+        if mupdf.pdf_is_array(contents):
+            n = mupdf.pdf_array_len(contents)
+            for i in range(n):
+                icont = mupdf.pdf_array_get(contents, i)
+                xref = mupdf.pdf_to_num(icont)
+                ret.append(xref)
+        elif contents.m_internal:
+            xref = mupdf.pdf_to_num(contents)
+            ret.append( xref)
+        return ret
+
+    def get_displaylist(self, annots=1):
+        '''
+        Make a DisplayList from the page for Pixmap generation.
+
+        Include (default) or exclude annotations.
+        '''
+        CheckParent(self)
+        if annots:
+            dl = mupdf.fz_new_display_list_from_page(self.this)
+        else:
+            dl = mupdf.fz_new_display_list_from_page_contents(self.this)
+        return DisplayList(dl)
+
+    def get_drawings(self, extended: bool=False) -> list:
+        """Retrieve vector graphics. The extended version includes clips.
+
+        Note:
+        For greater comfort, this method converts point-likes, rect-likes, quad-likes
+        of the C version to respective Point / Rect / Quad objects.
+        It also adds default items that are missing in original path types.
+        """
+        allkeys = (
+                'closePath',
+                'fill',
+                'color',
+                'width',
+                'lineCap',
+                'lineJoin',
+                'dashes',
+                'stroke_opacity',
+                'fill_opacity',
+                'even_odd',
+                )
+        val = self.get_cdrawings(extended=extended)
+        for i in range(len(val)):
+            npath = val[i]
+            if not npath["type"].startswith("clip"):
+                npath["rect"] = Rect(npath["rect"])
+            else:
+                npath["scissor"] = Rect(npath["scissor"])
+            if npath["type"]!="group":
+                items = npath["items"]
+                newitems = []
+                for item in items:
+                    cmd = item[0]
+                    rest = item[1:]
+                    if  cmd == "re":
+                        item = ("re", Rect(rest[0]).normalize(), rest[1])
+                    elif cmd == "qu":
+                        item = ("qu", Quad(rest[0]))
+                    else:
+                        item = tuple([cmd] + [Point(i) for i in rest])
+                    newitems.append(item)
+                npath["items"] = newitems
+            if npath['type'] in ('f', 's'):
+                for k in allkeys:
+                    npath[k] = npath.get(k)
+
+            val[i] = npath
+        return val
+
+        class Drawpath(object):
+            """Reflects a path dictionary from get_cdrawings()."""
+            def __init__(self, **args):
+                self.__dict__.update(args)
+        
+        class Drawpathlist(object):
+            """List of Path objects representing get_cdrawings() output."""
+            def __getitem__(self, item):
+                return self.paths.__getitem__(item)
+
+            def __init__(self):
+                self.paths = []
+                self.path_count = 0
+                self.group_count = 0
+                self.clip_count = 0
+                self.fill_count = 0
+                self.stroke_count = 0
+                self.fillstroke_count = 0
+
+            def __len__(self):
+                return self.paths.__len__()
+
+            def append(self, path):
+                self.paths.append(path)
+                self.path_count += 1
+                if path.type == "clip":
+                    self.clip_count += 1
+                elif path.type == "group":
+                    self.group_count += 1
+                elif path.type == "f":
+                    self.fill_count += 1
+                elif path.type == "s":
+                    self.stroke_count += 1
+                elif path.type == "fs":
+                    self.fillstroke_count += 1
+
+            def clip_parents(self, i):
+                """Return list of parent clip paths.
+
+                Args:
+                    i: (int) return parents of this path.
+                Returns:
+                    List of the clip parents."""
+                if i >= self.path_count:
+                    raise IndexError("bad path index")
+                while i < 0:
+                    i += self.path_count
+                lvl = self.paths[i].level
+                clips = list(  # clip paths before identified one
+                    reversed(
+                        [
+                            p
+                            for p in self.paths[:i]
+                            if p.type == "clip" and p.level < lvl
+                        ]
+                    )
+                )
+                if clips == []:  # none found: empty list
+                    return []
+                nclips = [clips[0]]  # init return list
+                for p in clips[1:]:
+                    if p.level >= nclips[-1].level:
+                        continue  # only accept smaller clip levels
+                    nclips.append(p)
+                return nclips
+
+            def group_parents(self, i):
+                """Return list of parent group paths.
+
+                Args:
+                    i: (int) return parents of this path.
+                Returns:
+                    List of the group parents."""
+                if i >= self.path_count:
+                    raise IndexError("bad path index")
+                while i < 0:
+                    i += self.path_count
+                lvl = self.paths[i].level
+                groups = list(  # group paths before identified one
+                    reversed(
+                        [
+                            p
+                            for p in self.paths[:i]
+                            if p.type == "group" and p.level < lvl
+                        ]
+                    )
+                )
+                if groups == []:  # none found: empty list
+                    return []
+                ngroups = [groups[0]]  # init return list
+                for p in groups[1:]:
+                    if p.level >= ngroups[-1].level:
+                        continue  # only accept smaller group levels
+                    ngroups.append(p)
+                return ngroups
+
+        def get_lineart(self) -> object:
+            """Get page drawings paths.
+
+            Note:
+            For greater comfort, this method converts point-like, rect-like, quad-like
+            tuples of the C version to respective Point / Rect / Quad objects.
+            Also adds default items that are missing in original path types.
+            In contrast to get_drawings(), this output is an object.
+            """
+
+            val = self.get_cdrawings(extended=True)
+            paths = self.Drawpathlist()
+            for path in val:
+                npath = self.Drawpath(**path)
+                if npath.type != "clip":
+                    npath.rect = Rect(path["rect"])
+                else:
+                    npath.scissor = Rect(path["scissor"])
+                if npath.type != "group":
+                    items = path["items"]
+                    newitems = []
+                    for item in items:
+                        cmd = item[0]
+                        rest = item[1:]
+                        if  cmd == "re":
+                            item = ("re", Rect(rest[0]).normalize(), rest[1])
+                        elif cmd == "qu":
+                            item = ("qu", Quad(rest[0]))
+                        else:
+                            item = tuple([cmd] + [Point(i) for i in rest])
+                        newitems.append(item)
+                    npath.items = newitems
+                
+                if npath.type == "f":
+                    npath.stroke_opacity = None
+                    npath.dashes = None
+                    npath.line_join = None
+                    npath.line_cap = None
+                    npath.color = None
+                    npath.width = None
+
+                paths.append(npath)
+
+            val = None
+            return paths
+
+    def remove_rotation(self):
+        """Set page rotation to 0 while maintaining visual appearance."""
+        rot = self.rotation  # normalized rotation value
+        if rot == 0:
+            return  Identity # nothing to do
+
+        # need to derotate the page's content
+        mb = self.mediabox  # current mediabox
+
+        if rot == 90:
+            # before derotation, shift content horizontally
+            mat0 = Matrix(1, 0, 0, 1, mb.y1 - mb.x1 - mb.x0 - mb.y0, 0)
+        elif rot == 270:
+            # before derotation, shift content vertically
+            mat0 = Matrix(1, 0, 0, 1, 0, mb.x1 - mb.y1 - mb.y0 - mb.x0)
+        else:  # rot = 180
+            mat0 = Matrix(1, 0, 0, 1, -2 * mb.x0, -2 * mb.y0)
+
+        # prefix with derotation matrix
+        mat = mat0 * self.derotation_matrix
+        cmd = _format_g(tuple(mat)) + ' cm '
+        cmd = cmd.encode('utf8')
+        _ = TOOLS._insert_contents(self, cmd, False)  # prepend to page contents
+
+        # swap x- and y-coordinates
+        if rot in (90, 270):
+            x0, y0, x1, y1 = mb
+            mb.x0 = y0
+            mb.y0 = x0
+            mb.x1 = y1
+            mb.y1 = x1
+            self.set_mediabox(mb)
+
+        self.set_rotation(0)
+        rot = ~mat  # inverse of the derotation matrix
+
+        for annot in self.annots():  # modify rectangles of annotations
+            r = annot.rect * rot
+            # TODO: only try to set rectangle for applicable annot types
+            annot.set_rect(r)
+        for link in self.get_links():  # modify 'from' rectangles of links
+            r = link["from"] * rot
+            self.delete_link(link)
+            link["from"] = r
+            try:  # invalid links remain deleted
+                self.insert_link(link)
+            except Exception:
+                pass
+        for widget in self.widgets():  # modify field rectangles
+            r = widget.rect * rot
+            widget.rect = r
+            widget.update()
+        return rot  # the inverse of the generated derotation matrix
+
+    def cluster_drawings(
+        self, clip=None, drawings=None, x_tolerance: float = 3, y_tolerance: float = 3,
+        final_filter: bool = True,
+    ) -> list:
+        """Join rectangles of neighboring vector graphic items.
+
+        Args:
+            clip: optional rect-like to restrict the page area to consider.
+            drawings: (optional) output of a previous "get_drawings()".
+            x_tolerance: horizontal neighborhood threshold.
+            y_tolerance: vertical neighborhood threshold.
+
+        Notes:
+            Vector graphics (also called line-art or drawings) usually consist
+            of independent items like rectangles, lines or curves to jointly
+            form table grid lines or bar, line, pie charts and similar.
+            This method identifies rectangles wrapping these disparate items.
+
+        Returns:
+            A list of Rect items, each wrapping line-art items that are close
+            enough to be considered forming a common vector graphic.
+            Only "significant" rectangles will be returned, i.e. having both,
+            width and height larger than the tolerance values.
+        """
+        CheckParent(self)
+        parea = self.rect  # the default clipping area
+        if clip is not None:
+            parea = Rect(clip)
+        delta_x = x_tolerance  # shorter local name
+        delta_y = y_tolerance  # shorter local name
+        if drawings is None:  # if we cannot re-use a previous output
+            drawings = self.get_drawings()
+
+        def are_neighbors(r1, r2):
+            """Detect whether r1, r2 are "neighbors".
+
+            Items r1, r2 are called neighbors if the minimum distance between
+            their points is less-equal delta.
+
+            Both parameters must be (potentially invalid) rectangles.
+            """
+            # normalize rectangles as needed
+            rr1_x0, rr1_x1 = (r1.x0, r1.x1) if r1.x1 > r1.x0 else (r1.x1, r1.x0)
+            rr1_y0, rr1_y1 = (r1.y0, r1.y1) if r1.y1 > r1.y0 else (r1.y1, r1.y0)
+            rr2_x0, rr2_x1 = (r2.x0, r2.x1) if r2.x1 > r2.x0 else (r2.x1, r2.x0)
+            rr2_y0, rr2_y1 = (r2.y0, r2.y1) if r2.y1 > r2.y0 else (r2.y1, r2.y0)
+            if (
+                0
+                or rr1_x1 < rr2_x0 - delta_x
+                or rr1_x0 > rr2_x1 + delta_x
+                or rr1_y1 < rr2_y0 - delta_y
+                or rr1_y0 > rr2_y1 + delta_y
+            ):
+                # Rects do not overlap.
+                return False
+            else:
+                # Rects overlap.
+                return True
+
+        # exclude graphics not contained in the clip
+        paths = [
+            p
+            for p in drawings
+            if 1
+            and p["rect"].x0 >= parea.x0
+            and p["rect"].x1 <= parea.x1
+            and p["rect"].y0 >= parea.y0
+            and p["rect"].y1 <= parea.y1
+        ]
+
+        # list of all vector graphic rectangles
+        prects = sorted([p["rect"] for p in paths], key=lambda r: (r.y1, r.x0))
+
+        new_rects = []  # the final list of the joined rectangles
+
+        # -------------------------------------------------------------------------
+        # The strategy is to identify and join all rects that are neighbors
+        # -------------------------------------------------------------------------
+        while prects:  # the algorithm will empty this list
+            r = +prects[0]  # copy of first rectangle
+            repeat = True
+            while repeat:
+                repeat = False
+                for i in range(len(prects) - 1, 0, -1):  # from back to front
+                    if are_neighbors(prects[i], r):
+                        r |= prects[i].tl  # include in first rect
+                        r |= prects[i].br  # include in first rect
+                        del prects[i]  # delete this rect
+                        repeat = True
+
+            new_rects.append(r)
+            del prects[0]
+            prects = sorted(set(prects), key=lambda r: (r.y1, r.x0))
+
+        new_rects = sorted(set(new_rects), key=lambda r: (r.y1, r.x0))
+        if not final_filter:
+            return new_rects
+        return [r for r in new_rects if r.width > delta_x and r.height > delta_y]
+
+    def get_fonts(self, full=False):
+        """List of fonts defined in the page object."""
+        CheckParent(self)
+        return self.parent.get_page_fonts(self.number, full=full)
+
+    def get_image_bbox(self, name, transform=0):
+        """Get rectangle occupied by image 'name'.
+
+        'name' is either an item of the image list, or the referencing
+        name string - elem[7] of the resp. item.
+        Option 'transform' also returns the image transformation matrix.
+        """
+        CheckParent(self)
+        doc = self.parent
+        if doc.is_closed or doc.is_encrypted:
+            raise ValueError('document closed or encrypted')
+
+        inf_rect = Rect(1, 1, -1, -1)
+        null_mat = Matrix()
+        if transform:
+            rc = (inf_rect, null_mat)
+        else:
+            rc = inf_rect
+
+        if type(name) in (list, tuple):
+            if not type(name[-1]) is int:
+                raise ValueError('need item of full page image list')
+            item = name
+        else:
+            imglist = [i for i in doc.get_page_images(self.number, True) if name == i[7]]
+            if len(imglist) == 1:
+                item = imglist[0]
+            elif imglist == []:
+                raise ValueError('bad image name')
+            else:
+                raise ValueError("found multiple images named '%s'." % name)
+        xref = item[-1]
+        if xref != 0 or transform:
+            try:
+                return self.get_image_rects(item, transform=transform)[0]
+            except Exception:
+                exception_info()
+                return inf_rect
+        pdf_page = self._pdf_page()
+        val = JM_image_reporter(pdf_page)
+
+        if not bool(val):
+            return rc
+
+        for v in val:
+            if v[0] != item[-3]:
+                continue
+            q = Quad(v[1])
+            bbox = q.rect
+            if transform == 0:
+                rc = bbox
+                break
+
+            hm = Matrix(util_hor_matrix(q.ll, q.lr))
+            h = abs(q.ll - q.ul)
+            w = abs(q.ur - q.ul)
+            m0 = Matrix(1 / w, 0, 0, 1 / h, 0, 0)
+            m = ~(hm * m0)
+            rc = (bbox, m)
+            break
+        val = rc
+
+        return val
+
+    def get_images(self, full=False):
+        """List of images defined in the page object."""
+        CheckParent(self)
+        return self.parent.get_page_images(self.number, full=full)
+
+    def get_oc_items(self) -> list:
+        """Get OCGs and OCMDs used in the page's contents.
+
+        Returns:
+            List of items (name, xref, type), where type is one of "ocg" / "ocmd",
+            and name is the property name.
+        """
+        rc = []
+        for pname, xref in self._get_resource_properties():
+            text = self.parent.xref_object(xref, compressed=True)
+            if "/Type/OCG" in text:
+                octype = "ocg"
+            elif "/Type/OCMD" in text:
+                octype = "ocmd"
+            else:
+                continue
+            rc.append((pname, xref, octype))
+        return rc
+
+    def get_svg_image(self, matrix=None, text_as_path=1):
+        """Make SVG image from page."""
+        CheckParent(self)
+        mediabox = mupdf.fz_bound_page(self.this)
+        ctm = JM_matrix_from_py(matrix)
+        tbounds = mediabox
+        text_option = mupdf.FZ_SVG_TEXT_AS_PATH if text_as_path == 1 else mupdf.FZ_SVG_TEXT_AS_TEXT
+        tbounds = mupdf.fz_transform_rect(tbounds, ctm)
+
+        res = mupdf.fz_new_buffer(1024)
+        out = mupdf.FzOutput(res)
+        dev = mupdf.fz_new_svg_device(
+                out,
+                tbounds.x1-tbounds.x0,  # width
+                tbounds.y1-tbounds.y0,  # height
+                text_option,
+                1,
+                )
+        mupdf.fz_run_page(self.this, dev, ctm, mupdf.FzCookie())
+        mupdf.fz_close_device(dev)
+        out.fz_close_output()
+        text = JM_EscapeStrFromBuffer(res)
+        return text
+
+    def get_textbox(
+            page: Page,
+            rect: rect_like,
+            textpage=None,  #: TextPage = None,
+            ) -> str:
+        tp = textpage
+        if tp is None:
+            tp = page.get_textpage()
+        elif getattr(tp, "parent") != page:
+            raise ValueError("not a textpage of this page")
+        rc = tp.extractTextbox(rect)
+        if textpage is None:
+            del tp
+        return rc
+
+    def get_textpage(self, clip: rect_like = None, flags: int = 0, matrix=None) -> "TextPage":
+        CheckParent(self)
+        if matrix is None:
+            matrix = Matrix(1, 1)
+        old_rotation = self.rotation
+        if old_rotation != 0:
+            self.set_rotation(0)
+        try:
+            textpage = self._get_textpage(clip, flags=flags, matrix=matrix)
+        finally:
+            if old_rotation != 0:
+                self.set_rotation(old_rotation)
+        textpage = TextPage(textpage)
+        textpage.parent = weakref.proxy(self)
+        return textpage
+
+    def get_texttrace(self):
+
+        CheckParent(self)
+        old_rotation = self.rotation
+        if old_rotation != 0:
+            self.set_rotation(0)
+        page = self.this
+        rc = []
+        if g_use_extra:
+            dev = extra.JM_new_texttrace_device(rc)
+        else:
+            dev = JM_new_texttrace_device(rc)
+        prect = mupdf.fz_bound_page(page)
+        dev.ptm = mupdf.FzMatrix(1, 0, 0, -1, 0, prect.y1)
+        mupdf.fz_run_page(page, dev, mupdf.FzMatrix(), mupdf.FzCookie())
+        mupdf.fz_close_device(dev)
+
+        if old_rotation != 0:
+            self.set_rotation(old_rotation)
+        return rc
+
+    def get_xobjects(self):
+        """List of xobjects defined in the page object."""
+        CheckParent(self)
+        return self.parent.get_page_xobjects(self.number)
+
+    def insert_font(self, fontname="helv", fontfile=None, fontbuffer=None,
+                   set_simple=False, wmode=0, encoding=0):
+        doc = self.parent
+        if doc is None:
+            raise ValueError("orphaned object: parent is None")
+        idx = 0
+
+        if fontname.startswith("/"):
+            fontname = fontname[1:]
+        inv_chars = INVALID_NAME_CHARS.intersection(fontname)
+        if inv_chars != set():
+            raise ValueError(f"bad fontname chars {inv_chars}")
+
+        font = CheckFont(self, fontname)
+        if font is not None:                    # font already in font list of page
+            xref = font[0]                      # this is the xref
+            if CheckFontInfo(doc, xref):        # also in our document font list?
+                return xref                     # yes: we are done
+            # need to build the doc FontInfo entry - done via get_char_widths
+            doc.get_char_widths(xref)
+            return xref
+
+        #--------------------------------------------------------------------------
+        # the font is not present for this page
+        #--------------------------------------------------------------------------
+
+        bfname = Base14_fontdict.get(fontname.lower(), None) # BaseFont if Base-14 font
+
+        serif = 0
+        CJK_number = -1
+        CJK_list_n = ["china-t", "china-s", "japan", "korea"]
+        CJK_list_s = ["china-ts", "china-ss", "japan-s", "korea-s"]
+
+        try:
+            CJK_number = CJK_list_n.index(fontname)
+            serif = 0
+        except Exception:
+            # Verbose in PyMuPDF/tests.
+            if g_exceptions_verbose > 1:    exception_info()
+            pass
+
+        if CJK_number < 0:
+            try:
+                CJK_number = CJK_list_s.index(fontname)
+                serif = 1
+            except Exception:
+                # Verbose in PyMuPDF/tests.
+                if g_exceptions_verbose > 1:    exception_info()
+                pass
+
+        if fontname.lower() in fitz_fontdescriptors.keys():
+            import pymupdf_fonts
+            fontbuffer = pymupdf_fonts.myfont(fontname)  # make a copy
+            del pymupdf_fonts
+
+        # install the font for the page
+        if fontfile is not None:
+            if type(fontfile) is str:
+                fontfile_str = fontfile
+            elif hasattr(fontfile, "absolute"):
+                fontfile_str = str(fontfile)
+            elif hasattr(fontfile, "name"):
+                fontfile_str = fontfile.name
+            else:
+                raise ValueError("bad fontfile")
+        else:
+            fontfile_str = None
+        val = self._insertFont(fontname, bfname, fontfile_str, fontbuffer, set_simple, idx,
+                               wmode, serif, encoding, CJK_number)
+
+        if not val:                   # did not work, error return
+            return val
+
+        xref = val[0]                 # xref of installed font
+        fontdict = val[1]
+
+        if CheckFontInfo(doc, xref):  # check again: document already has this font
+            return xref               # we are done
+
+        # need to create document font info
+        doc.get_char_widths(xref, fontdict=fontdict)
+        return xref
+
+    @property
+    def is_wrapped(self):
+        """Check if /Contents is in a balanced graphics state."""
+        return self._count_q_balance() == (0, 0)
+
+    @property
+    def language(self):
+        """Page language."""
+        pdfpage = _as_pdf_page(self.this, required=False)
+        if not pdfpage.m_internal:
+            return
+        lang = mupdf.pdf_dict_get_inheritable(pdfpage.obj(), PDF_NAME('Lang'))
+        if not lang.m_internal:
+            return
+        return mupdf.pdf_to_str_buf(lang)
+
+    def links(self, kinds=None):
+        """ Generator over the links of a page.
+
+        Args:
+            kinds: (list) link kinds to subselect from. If none,
+                   all links are returned. E.g. kinds=[LINK_URI]
+                   will only yield URI links.
+        """
+        all_links = self.get_links()
+        for link in all_links:
+            if kinds is None or link["kind"] in kinds:
+                yield (link)
+
+    def load_annot(self, ident: typing.Union[str, int]) -> Annot:
+        """Load an annot by name (/NM key) or xref.
+
+        Args:
+            ident: identifier, either name (str) or xref (int).
+        """
+        CheckParent(self)
+        if type(ident) is str:
+            xref = 0
+            name = ident
+        elif type(ident) is int:
+            xref = ident
+            name = None
+        else:
+            raise ValueError("identifier must be a string or integer")
+        val = self._load_annot(name, xref)
+        if not val:
+            return val
+        val.thisown = True
+        val.parent = weakref.proxy(self)
+        self._annot_refs[id(val)] = val
+        return val
+
+    def load_links(self):
+        """Get first Link."""
+        CheckParent(self)
+        val = mupdf.fz_load_links( self.this)
+        if not val.m_internal:
+            return
+        val = Link( val)
+        val.thisown = True
+        val.parent = weakref.proxy(self) # owning page object
+        self._annot_refs[id(val)] = val
+        val.xref = 0
+        val.id = ""
+        if self.parent.is_pdf:
+            xrefs = self.annot_xrefs()
+            xrefs = [x for x in xrefs if x[1] == mupdf.PDF_ANNOT_LINK]
+            if xrefs:
+                link_id = xrefs[0]
+                val.xref = link_id[0]
+                val.id = link_id[2]
+        else:
+            val.xref = 0
+            val.id = ""
+        return val
+
+    #----------------------------------------------------------------
+    # page load widget by xref
+    #----------------------------------------------------------------
+    def load_widget( self, xref):
+        """Load a widget by its xref."""
+        CheckParent(self)
+
+        page = _as_pdf_page(self.this)
+        annot = JM_get_widget_by_xref( page, xref)
+        #log( '{=type(annot)}')
+        val = annot
+        if not val:
+            return val
+        val.thisown = True
+        val.parent = weakref.proxy(self)
+        self._annot_refs[id(val)] = val
+        widget = Widget()
+        TOOLS._fill_widget(val, widget)
+        val = widget
+        return val
+
+    @property
+    def mediabox(self):
+        """The MediaBox."""
+        CheckParent(self)
+        page = self._pdf_page(required=False)
+        if not page.m_internal:
+            rect = mupdf.fz_bound_page( self.this)
+        else:
+            rect = JM_mediabox( page.obj())
+        return Rect(rect)
+
+    @property
+    def mediabox_size(self):
+        return Point(self.mediabox.x1, self.mediabox.y1)
+
+    #@property
+    #def parent( self):
+    #    assert self._parent
+    #    if self._parent:
+    #        return self._parent
+    #    return Document( self.this.document())
+
+    def read_contents(self):
+        """All /Contents streams concatenated to one bytes object."""
+        return TOOLS._get_all_contents(self)
+
+    def refresh(self):
+        """Refresh page after link/annot/widget updates."""
+        CheckParent(self)
+        doc = self.parent
+        page = doc.reload_page(self)
+        # fixme this looks wrong.
+        self.this = page
+
+    @property
+    def rotation(self):
+        """Page rotation."""
+        CheckParent(self)
+        page = _as_pdf_page(self.this, required=0)
+        if not page.m_internal:
+            return 0
+        return JM_page_rotation(page)
+
+    @property
+    def rotation_matrix(self) -> Matrix:
+        """Reflects page rotation."""
+        return Matrix(TOOLS._rotate_matrix(self))
+
+    def run(self, dw, m):
+        """Run page through a device.
+        dw: DeviceWrapper
+        """
+        CheckParent(self)
+        mupdf.fz_run_page(self.this, dw.device, JM_matrix_from_py(m), mupdf.FzCookie())
+
+    def set_artbox(self, rect):
+        """Set the ArtBox."""
+        return self._set_pagebox("ArtBox", rect)
+
+    def set_bleedbox(self, rect):
+        """Set the BleedBox."""
+        return self._set_pagebox("BleedBox", rect)
+
+    def set_contents(self, xref):
+        """Set object at 'xref' as the page's /Contents."""
+        CheckParent(self)
+        doc = self.parent
+        if doc.is_closed:
+            raise ValueError("document closed")
+        if not doc.is_pdf:
+            raise ValueError("is no PDF")
+        if xref not in range(1, doc.xref_length()):
+            raise ValueError("bad xref")
+        if not doc.xref_is_stream(xref):
+            raise ValueError("xref is no stream")
+        doc.xref_set_key(self.xref, "Contents", "%i 0 R" % xref)
+
+    def set_cropbox(self, rect):
+        """Set the CropBox. Will also change Page.rect."""
+        return self._set_pagebox("CropBox", rect)
+
+    def set_language(self, language=None):
+        """Set PDF page default language."""
+        CheckParent(self)
+        pdfpage = _as_pdf_page(self.this)
+        if not language:
+            mupdf.pdf_dict_del(pdfpage.obj(), PDF_NAME('Lang'))
+        else:
+            lang = mupdf.fz_text_language_from_string(language)
+            assert hasattr(mupdf, 'fz_string_from_text_language2')
+            mupdf.pdf_dict_put_text_string(
+                    pdfpage.obj,
+                    PDF_NAME('Lang'),
+                    mupdf.fz_string_from_text_language2(lang)
+                    )
+
+    def set_mediabox(self, rect):
+        """Set the MediaBox."""
+        CheckParent(self)
+        page = self._pdf_page()
+        mediabox = JM_rect_from_py(rect)
+        if (mupdf.fz_is_empty_rect(mediabox)
+                or mupdf.fz_is_infinite_rect(mediabox)
+                ):
+            raise ValueError( MSG_BAD_RECT)
+        mupdf.pdf_dict_put_rect( page.obj(), PDF_NAME('MediaBox'), mediabox)
+        mupdf.pdf_dict_del( page.obj(), PDF_NAME('CropBox'))
+        mupdf.pdf_dict_del( page.obj(), PDF_NAME('ArtBox'))
+        mupdf.pdf_dict_del( page.obj(), PDF_NAME('BleedBox'))
+        mupdf.pdf_dict_del( page.obj(), PDF_NAME('TrimBox'))
+
+    def set_rotation(self, rotation):
+        """Set page rotation."""
+        CheckParent(self)
+        page = _as_pdf_page(self.this)
+        rot = JM_norm_rotation(rotation)
+        mupdf.pdf_dict_put_int( page.obj(), PDF_NAME('Rotate'), rot)
+
+    def set_trimbox(self, rect):
+        """Set the TrimBox."""
+        return self._set_pagebox("TrimBox", rect)
+
+    @property
+    def transformation_matrix(self):
+        """Page transformation matrix."""
+        CheckParent(self)
+
+        ctm = mupdf.FzMatrix()
+        page = self._pdf_page(required=False)
+        if not page.m_internal:
+            return JM_py_from_matrix(ctm)
+        mediabox = mupdf.FzRect(mupdf.FzRect.Fixed_UNIT)    # fixme: original code passed mediabox=NULL.
+        mupdf.pdf_page_transform(page, mediabox, ctm)
+        val = JM_py_from_matrix(ctm)
+
+        if self.rotation % 360 == 0:
+            val = Matrix(val)
+        else:
+            val = Matrix(1, 0, 0, -1, 0, self.cropbox.height)
+        return val
+
+    @property
+    def trimbox(self):
+        """The TrimBox"""
+        rect = self._other_box("TrimBox")
+        if rect is None:
+            return self.cropbox
+        mb = self.mediabox
+        return Rect(rect[0], mb.y1 - rect[3], rect[2], mb.y1 - rect[1])
+
+    def widgets(self, types=None):
+        """ Generator over the widgets of a page.
+
+        Args:
+            types: (list) field types to subselect from. If none,
+                    all fields are returned. E.g. types=[PDF_WIDGET_TYPE_TEXT]
+                    will only yield text fields.
+        """
+        #for a in self.annot_xrefs():
+        #    log( '{a=}')
+        widget_xrefs = [a[0] for a in self.annot_xrefs() if a[1] == mupdf.PDF_ANNOT_WIDGET]
+        #log(f'widgets(): {widget_xrefs=}')
+        for xref in widget_xrefs:
+            widget = self.load_widget(xref)
+            if types is None or widget.field_type in types:
+                yield (widget)
+
+    def wrap_contents(self):
+        """Ensure page is in a balanced graphics state."""
+        push, pop = self._count_q_balance()  # count missing "q"/"Q" commands
+        if push > 0:  # prepend required push commands
+            prepend = b"q\n" * push
+            TOOLS._insert_contents(self, prepend, False)
+        if pop > 0:  # append required pop commands
+            append = b"\nQ" * pop + b"\n"
+            TOOLS._insert_contents(self, append, True)
+
+    @property
+    def xref(self):
+        """PDF xref number of page."""
+        CheckParent(self)
+        return self.parent.page_xref(self.number)
+
+    rect = property(bound, doc="page rectangle")
+
+
+class Pixmap:
+
+    def __init__(self, *args):
+        """
+        Pixmap(colorspace, irect, alpha) - empty pixmap.
+        Pixmap(colorspace, src) - copy changing colorspace.
+        Pixmap(src, width, height,[clip]) - scaled copy, float dimensions.
+        Pixmap(src, alpha=1) - copy and add or drop alpha channel.
+        Pixmap(filename) - from an image in a file.
+        Pixmap(image) - from an image in memory (bytes).
+        Pixmap(colorspace, width, height, samples, alpha) - from samples data.
+        Pixmap(PDFdoc, xref) - from an image at xref in a PDF document.
+        """
+        # Cache for property `self.samples_mv`. Set here so __del_() sees it if
+        # we raise.
+        #
+        self._samples_mv = None
+
+        # 2024-01-16: Experimental support for a memory-view of the underlying
+        # data.  Doesn't seem to make much difference to Pixmap.set_pixel() so
+        # not currently used.
+        self._memory_view = None
+        
+        if 0:
+            pass
+
+        elif args_match(args,
+                (Colorspace, mupdf.FzColorspace),
+                (mupdf.FzRect, mupdf.FzIrect, IRect, Rect, tuple)
+                ):
+            # create empty pixmap with colorspace and IRect
+            cs, rect = args
+            alpha = 0
+            pm = mupdf.fz_new_pixmap_with_bbox(cs, JM_irect_from_py(rect), mupdf.FzSeparations(0), alpha)
+            self.this = pm
+
+        elif args_match(args,
+                (Colorspace, mupdf.FzColorspace),
+                (mupdf.FzRect, mupdf.FzIrect, IRect, Rect, tuple),
+                (int, bool)
+                ):
+            # create empty pixmap with colorspace and IRect
+            cs, rect, alpha = args
+            pm = mupdf.fz_new_pixmap_with_bbox(cs, JM_irect_from_py(rect), mupdf.FzSeparations(0), alpha)
+            self.this = pm
+
+        elif args_match(args, (Colorspace, mupdf.FzColorspace, type(None)), (Pixmap, mupdf.FzPixmap)):
+            # copy pixmap, converting colorspace
+            cs, spix = args
+            if isinstance(cs, Colorspace):
+                cs = cs.this
+            elif cs is None:
+                cs = mupdf.FzColorspace(None)
+            if isinstance(spix, Pixmap):
+                spix = spix.this
+            if not mupdf.fz_pixmap_colorspace(spix).m_internal:
+                raise ValueError( "source colorspace must not be None")
+            
+            if cs.m_internal:
+                self.this = mupdf.fz_convert_pixmap(
+                        spix,
+                        cs,
+                        mupdf.FzColorspace(),
+                        mupdf.FzDefaultColorspaces(None),
+                        mupdf.FzColorParams(),
+                        1
+                        )
+            else:
+                self.this = mupdf.fz_new_pixmap_from_alpha_channel( spix)
+                if not self.this.m_internal:
+                    raise RuntimeError( MSG_PIX_NOALPHA)
+
+        elif args_match(args, (Pixmap, mupdf.FzPixmap), (Pixmap, mupdf.FzPixmap)):
+            # add mask to a pixmap w/o alpha channel
+            spix, mpix = args
+            if isinstance(spix, Pixmap):
+                spix = spix.this
+            if isinstance(mpix, Pixmap):
+                mpix = mpix.this
+            spm = spix
+            mpm = mpix
+            if not spix.m_internal: # intercept NULL for spix: make alpha only pix
+                dst = mupdf.fz_new_pixmap_from_alpha_channel(mpm)
+                if not dst.m_internal:
+                    raise RuntimeError( MSG_PIX_NOALPHA)
+            else:
+                dst = mupdf.fz_new_pixmap_from_color_and_mask(spm, mpm)
+            self.this = dst
+
+        elif (args_match(args, (Pixmap, mupdf.FzPixmap), (float, int), (float, int), None) or
+             args_match(args, (Pixmap, mupdf.FzPixmap), (float, int), (float, int))):
+            # create pixmap as scaled copy of another one
+            if len(args) == 3:
+                spix, w, h = args
+                bbox = mupdf.FzIrect(mupdf.fz_infinite_irect)
+            else:
+                spix, w, h, clip = args
+                bbox = JM_irect_from_py(clip)
+        
+            src_pix = spix.this if isinstance(spix, Pixmap) else spix
+            if not mupdf.fz_is_infinite_irect(bbox):
+                pm = mupdf.fz_scale_pixmap(src_pix, src_pix.x(), src_pix.y(), w, h, bbox)
+            else:
+                pm = mupdf.fz_scale_pixmap(src_pix, src_pix.x(), src_pix.y(), w, h, mupdf.FzIrect(mupdf.fz_infinite_irect))
+            self.this = pm
+
+        elif args_match(args, str, (Pixmap, mupdf.FzPixmap)) and args[0] == 'raw':
+            # Special raw construction where we set .this directly.
+            _, pm = args
+            if isinstance(pm, Pixmap):
+                pm = pm.this
+            self.this = pm
+
+        elif args_match(args, (Pixmap, mupdf.FzPixmap), (int, None)):
+            # Pixmap(struct Pixmap *spix, int alpha=1)
+            # copy pixmap & add / drop the alpha channel
+            spix = args[0]
+            alpha = args[1] if len(args) == 2 else 1
+            src_pix = spix.this if isinstance(spix, Pixmap) else spix
+            if not _INRANGE(alpha, 0, 1):
+                raise ValueError( "bad alpha value")
+            cs = mupdf.fz_pixmap_colorspace(src_pix)
+            if not cs.m_internal and not alpha:
+                raise ValueError( "cannot drop alpha for 'NULL' colorspace")
+            seps = mupdf.FzSeparations()
+            n = mupdf.fz_pixmap_colorants(src_pix)
+            w = mupdf.fz_pixmap_width(src_pix)
+            h = mupdf.fz_pixmap_height(src_pix)
+            pm = mupdf.fz_new_pixmap(cs, w, h, seps, alpha)
+            pm.m_internal.x = src_pix.m_internal.x
+            pm.m_internal.y = src_pix.m_internal.y
+            pm.m_internal.xres = src_pix.m_internal.xres
+            pm.m_internal.yres = src_pix.m_internal.yres
+
+            # copy samples data ------------------------------------------
+            if 1:
+                # We use our pixmap_copy() to get best performance.
+                # test_pixmap.py:test_setalpha(): 3.9s t=0.0062
+                extra.pixmap_copy( pm.m_internal, src_pix.m_internal, n)
+            elif 1:
+                # Use memoryview.
+                # test_pixmap.py:test_setalpha(): 4.6 t=0.51
+                src_view = mupdf.fz_pixmap_samples_memoryview( src_pix)
+                pm_view = mupdf.fz_pixmap_samples_memoryview( pm)
+                if src_pix.alpha() == pm.alpha():   # identical samples
+                    #memcpy(tptr, sptr, w * h * (n + alpha));
+                    size = w * h * (n + alpha)
+                    pm_view[ 0 : size] = src_view[ 0 : size]
+                else:
+                    tptr = 0
+                    sptr = 0
+                    # This is a little faster than calling
+                    # pm.fz_samples_set(), but still quite slow. E.g. reduces
+                    # test_pixmap.py:test_setalpha() from 6.7s to 4.5s.
+                    #
+                    # t=0.53
+                    pm_stride = pm.stride()
+                    pm_n = pm.n()
+                    pm_alpha = pm.alpha()
+                    src_stride = src_pix.stride()
+                    src_n = src_pix.n()
+                    #log( '{=pm_stride pm_n src_stride src_n}')
+                    for y in range( h):
+                        for x in range( w):
+                            pm_i = pm_stride * y + pm_n * x
+                            src_i = src_stride * y + src_n * x
+                            pm_view[ pm_i : pm_i + n] = src_view[ src_i : src_i + n]
+                            if pm_alpha:
+                                pm_view[ pm_i + n] = 255
+            else:
+                # Copy individual bytes from Python. Very slow.
+                # test_pixmap.py:test_setalpha(): 6.89 t=2.601
+                if src_pix.alpha() == pm.alpha():   # identical samples
+                    #memcpy(tptr, sptr, w * h * (n + alpha));
+                    for i in range(w * h * (n + alpha)):
+                        mupdf.fz_samples_set(pm, i, mupdf.fz_samples_get(src_pix, i))
+                else:
+                    # t=2.56
+                    tptr = 0
+                    sptr = 0
+                    src_pix_alpha = src_pix.alpha()
+                    for i in range(w * h):
+                        #memcpy(tptr, sptr, n);
+                        for j in range(n):
+                            mupdf.fz_samples_set(pm, tptr + j, mupdf.fz_samples_get(src_pix, sptr + j))
+                        tptr += n
+                        if pm.alpha():
+                            mupdf.fz_samples_set(pm, tptr, 255)
+                            tptr += 1
+                        sptr += n + src_pix_alpha
+            self.this = pm
+
+        elif args_match(args, (mupdf.FzColorspace, Colorspace), int, int, None, (int, bool)):
+            # create pixmap from samples data
+            cs, w, h, samples, alpha = args
+            if isinstance(cs, Colorspace):
+                cs = cs.this
+                assert isinstance(cs, mupdf.FzColorspace)
+            n = mupdf.fz_colorspace_n(cs)
+            stride = (n + alpha) * w
+            seps = mupdf.FzSeparations()
+            pm = mupdf.fz_new_pixmap(cs, w, h, seps, alpha)
+
+            if isinstance( samples, (bytes, bytearray)):
+                #log('using mupdf.python_buffer_data()')
+                samples2 = mupdf.python_buffer_data(samples)
+                size = len(samples)
+            else:
+                res = JM_BufferFromBytes(samples)
+                if not res.m_internal:
+                    raise ValueError( "bad samples data")
+                size, c = mupdf.fz_buffer_storage(res)
+                samples2 = mupdf.python_buffer_data(samples) # raw swig proxy for `const unsigned char*`.
+            if stride * h != size:
+                raise ValueError( f"bad samples length {w=} {h=} {alpha=} {n=} {stride=} {size=}")
+            mupdf.ll_fz_pixmap_copy_raw( pm.m_internal, samples2)
+            self.this = pm
+
+        elif args_match(args, None):
+            # create pixmap from filename, file object, pathlib.Path or memory
+            imagedata, = args
+            name = 'name'
+            if hasattr(imagedata, "resolve"):
+                fname = imagedata.__str__()
+                if fname:
+                    img = mupdf.fz_new_image_from_file(fname)
+            elif hasattr(imagedata, name):
+                fname = imagedata.name
+                if fname:
+                    img = mupdf.fz_new_image_from_file(fname)
+            elif isinstance(imagedata, str):
+                img = mupdf.fz_new_image_from_file(imagedata)
+            else:
+                res = JM_BufferFromBytes(imagedata)
+                if not res.m_internal or not res.m_internal.len:
+                    raise ValueError( "bad image data")
+                img = mupdf.fz_new_image_from_buffer(res)
+
+            # Original code passed null for subarea and ctm, but that's not
+            # possible with MuPDF's python bindings. The equivalent is an
+            # infinite rect and identify matrix scaled by img.w() and img.h().
+            pm, w, h = mupdf.fz_get_pixmap_from_image(
+                    img,
+                    mupdf.FzIrect(FZ_MIN_INF_RECT, FZ_MIN_INF_RECT, FZ_MAX_INF_RECT, FZ_MAX_INF_RECT),
+                    mupdf.FzMatrix( img.w(), 0, 0, img.h(), 0, 0),
+                    )
+            xres, yres = mupdf.fz_image_resolution(img)
+            pm.m_internal.xres = xres
+            pm.m_internal.yres = yres
+            self.this = pm
+
+        elif args_match(args, (Document, mupdf.FzDocument), int):
+            # Create pixmap from PDF image identified by XREF number
+            doc, xref = args
+            pdf = _as_pdf_document(doc)
+            xreflen = mupdf.pdf_xref_len(pdf)
+            if not _INRANGE(xref, 1, xreflen-1):
+                raise ValueError( MSG_BAD_XREF)
+            ref = mupdf.pdf_new_indirect(pdf, xref, 0)
+            type_ = mupdf.pdf_dict_get(ref, PDF_NAME('Subtype'))
+            if (not mupdf.pdf_name_eq(type_, PDF_NAME('Image'))
+                    and not mupdf.pdf_name_eq(type_, PDF_NAME('Alpha'))
+                    and not mupdf.pdf_name_eq(type_, PDF_NAME('Luminosity'))
+                    ):
+                raise ValueError( MSG_IS_NO_IMAGE)
+            img = mupdf.pdf_load_image(pdf, ref)
+            # Original code passed null for subarea and ctm, but that's not
+            # possible with MuPDF's python bindings. The equivalent is an
+            # infinite rect and identify matrix scaled by img.w() and img.h().
+            pix, w, h = mupdf.fz_get_pixmap_from_image(
+                    img,
+                    mupdf.FzIrect(FZ_MIN_INF_RECT, FZ_MIN_INF_RECT, FZ_MAX_INF_RECT, FZ_MAX_INF_RECT),
+                    mupdf.FzMatrix(img.w(), 0, 0, img.h(), 0, 0),
+                    )
+            self.this = pix
+
+        else:
+            text = 'Unrecognised args for constructing Pixmap:\n'
+            for arg in args:
+                text += f'    {type(arg)}: {arg}\n'
+            raise Exception( text)
+
+    def __len__(self):
+        return self.size
+
+    def __repr__(self):
+        if not type(self) is Pixmap: return
+        if self.colorspace:
+            return "Pixmap(%s, %s, %s)" % (self.colorspace.this.m_internal.name, self.irect, self.alpha)
+        else:
+            return "Pixmap(%s, %s, %s)" % ('None', self.irect, self.alpha)
+
+    def _tobytes(self, format_, jpg_quality):
+        '''
+        Pixmap._tobytes
+        '''
+        pm = self.this
+        size = mupdf.fz_pixmap_stride(pm) * pm.h()
+        res = mupdf.fz_new_buffer(size)
+        out = mupdf.FzOutput(res)
+        if   format_ == 1:  mupdf.fz_write_pixmap_as_png(out, pm)
+        elif format_ == 2:  mupdf.fz_write_pixmap_as_pnm(out, pm)
+        elif format_ == 3:  mupdf.fz_write_pixmap_as_pam(out, pm)
+        elif format_ == 5:  mupdf.fz_write_pixmap_as_psd(out, pm)
+        elif format_ == 6:  mupdf.fz_write_pixmap_as_ps(out, pm)
+        elif format_ == 7:
+            mupdf.fz_write_pixmap_as_jpeg(out, pm, jpg_quality, 0)
+        else:
+            mupdf.fz_write_pixmap_as_png(out, pm)
+        out.fz_close_output()
+        barray = JM_BinFromBuffer(res)
+        return barray
+
+    def _writeIMG(self, filename, format_, jpg_quality):
+        pm = self.this
+        if   format_ == 1:  mupdf.fz_save_pixmap_as_png(pm, filename)
+        elif format_ == 2:  mupdf.fz_save_pixmap_as_pnm(pm, filename)
+        elif format_ == 3:  mupdf.fz_save_pixmap_as_pam(pm, filename)
+        elif format_ == 5:  mupdf.fz_save_pixmap_as_psd(pm, filename)
+        elif format_ == 6:  mupdf.fz_save_pixmap_as_ps(pm, filename)
+        elif format_ == 7:  mupdf.fz_save_pixmap_as_jpeg(pm, filename, jpg_quality)
+        else:               mupdf.fz_save_pixmap_as_png(pm, filename)
+
+    @property
+    def alpha(self):
+        """Indicates presence of alpha channel."""
+        return mupdf.fz_pixmap_alpha(self.this)
+
+    def clear_with(self, value=None, bbox=None):
+        """Fill all color components with same value."""
+        if value is None:
+            mupdf.fz_clear_pixmap(self.this)
+        elif bbox is None:
+            mupdf.fz_clear_pixmap_with_value(self.this, value)
+        else:
+            JM_clear_pixmap_rect_with_value(self.this, value, JM_irect_from_py(bbox))
+
+    def color_count(self, colors=0, clip=None):
+        '''
+        Return count of each color.
+        '''
+        pm = self.this
+        rc = JM_color_count( pm, clip)
+        if not colors:
+            return len( rc)
+        return rc
+
+    def color_topusage(self, clip=None):
+        """Return most frequent color and its usage ratio."""
+        allpixels = 0
+        cnt = 0
+        if clip is not None and self.irect in Rect(clip):
+            clip = self.irect
+        for pixel, count in self.color_count(colors=True,clip=clip).items():
+            allpixels += count
+            if count > cnt:
+                cnt = count
+                maxpixel = pixel
+        if not allpixels:
+            return (1, bytes([255] * self.n))
+        return (cnt / allpixels, maxpixel)
+
+    @property
+    def colorspace(self):
+        """Pixmap Colorspace."""
+        cs = Colorspace(mupdf.fz_pixmap_colorspace(self.this))
+        if cs.name == "None":
+            return None
+        return cs
+
+    def copy(self, src, bbox):
+        """Copy bbox from another Pixmap."""
+        pm = self.this
+        src_pix = src.this
+        if not mupdf.fz_pixmap_colorspace(src_pix):
+            raise ValueError( "cannot copy pixmap with NULL colorspace")
+        if pm.alpha() != src_pix.alpha():
+            raise ValueError( "source and target alpha must be equal")
+        mupdf.fz_copy_pixmap_rect(pm, src_pix, JM_irect_from_py(bbox), mupdf.FzDefaultColorspaces(None))
+
+    @property
+    def digest(self):
+        """MD5 digest of pixmap (bytes)."""
+        ret = mupdf.fz_md5_pixmap2(self.this)
+        return bytes(ret)
+
+    def gamma_with(self, gamma):
+        """Apply correction with some float.
+        gamma=1 is a no-op."""
+        if not mupdf.fz_pixmap_colorspace( self.this):
+            message_warning("colorspace invalid for function")
+            return
+        mupdf.fz_gamma_pixmap( self.this, gamma)
+
+    @property
+    def h(self):
+        """The height."""
+        return mupdf.fz_pixmap_height(self.this)
+
+    def invert_irect(self, bbox=None):
+        """Invert the colors inside a bbox."""
+        pm = self.this
+        if not mupdf.fz_pixmap_colorspace(pm).m_internal:
+            message_warning("ignored for stencil pixmap")
+            return False
+        r = JM_irect_from_py(bbox)
+        if mupdf.fz_is_infinite_irect(r):
+            mupdf.fz_invert_pixmap(pm)
+            return True
+        mupdf.fz_invert_pixmap_rect(pm, r)
+        return True
+
+    @property
+    def irect(self):
+        """Pixmap bbox - an IRect object."""
+        val = mupdf.fz_pixmap_bbox(self.this)
+        return JM_py_from_irect( val)
+
+    @property
+    def is_monochrome(self):
+        """Check if pixmap is monochrome."""
+        return mupdf.fz_is_pixmap_monochrome( self.this)
+
+    @property
+    def is_unicolor(self):
+        '''
+        Check if pixmap has only one color.
+        '''
+        pm = self.this
+        n = pm.n()
+        count = pm.w() * pm.h() * n
+        def _pixmap_read_samples(pm, offset, n):
+            ret = list()
+            for i in range(n):
+                ret.append(mupdf.fz_samples_get(pm, offset+i))
+            return ret
+        for offset in range( 0, count, n):
+            if offset == 0:
+                sample0 = _pixmap_read_samples( pm, 0, n)
+            else:
+                sample = _pixmap_read_samples( pm, offset, n)
+                if sample != sample0:
+                    return False
+        return True
+
+    @property
+    def n(self):
+        """The size of one pixel."""
+        if g_use_extra:
+            # Setting self.__class__.n gives a small reduction in overhead of
+            # test_general.py:test_2093, e.g. 1.4x -> 1.3x.
+            #return extra.pixmap_n(self.this)
+            def n2(self):
+                return extra.pixmap_n(self.this)
+            self.__class__.n = property(n2)
+            return self.n
+        return mupdf.fz_pixmap_components(self.this)
+
+    def pdfocr_save(self, filename, compress=1, language=None, tessdata=None):
+        '''
+        Save pixmap as an OCR-ed PDF page.
+        '''
+        tessdata = get_tessdata(tessdata)
+        opts = mupdf.FzPdfocrOptions()
+        opts.compress = compress
+        if language:
+            opts.language_set2( language)
+        if tessdata:
+            opts.datadir_set2( tessdata)
+        pix = self.this
+        if isinstance(filename, str):
+            mupdf.fz_save_pixmap_as_pdfocr( pix, filename, 0, opts)
+        else:
+            out = JM_new_output_fileptr( filename)
+            try:
+                mupdf.fz_write_pixmap_as_pdfocr( out, pix, opts)
+            finally:
+                out.fz_close_output()   # Avoid MuPDF warning.
+
+    def pdfocr_tobytes(self, compress=True, language="eng", tessdata=None):
+        """Save pixmap as an OCR-ed PDF page.
+
+        Args:
+            compress: (bool) compress, default 1 (True).
+            language: (str) language(s) occurring on page, default "eng" (English),
+                    multiples like "eng+ger" for English and German.
+            tessdata: (str) folder name of Tesseract's language support. If None
+                    we use environment variable TESSDATA_PREFIX or search for
+                    Tesseract installation.
+        Notes:
+            On failure, make sure Tesseract is installed and you have set
+             or environment variable "TESSDATA_PREFIX" to the folder
+            containing your Tesseract's language support data.
+        """
+        tessdata = get_tessdata(tessdata)
+        from io import BytesIO
+        bio = BytesIO()
+        self.pdfocr_save(bio, compress=compress, language=language, tessdata=tessdata)
+        return bio.getvalue()
+
+    def pil_image(self):
+        """Create a Pillow Image from the Pixmap."""
+        try:
+            from PIL import Image
+        except ImportError:
+            message("PIL/Pillow not installed")
+            raise
+
+        cspace = self.colorspace
+        if not cspace:
+            mode = "L"
+        elif cspace.n == 1:
+            mode = "L" if not self.alpha else "LA"
+        elif cspace.n == 3:
+            mode = "RGB" if not self.alpha else "RGBA"
+        else:
+            mode = "CMYK"
+
+        img = Image.frombytes(mode, (self.width, self.height), self.samples)
+        return img
+
+    def pil_save(self, *args, **kwargs):
+        """Write to image file using Pillow.
+
+        An intermediate PIL Image is created, and its "save" method is used
+        to store the image. See Pillow documentation to learn about the
+        meaning of possible positional and keyword parameters.
+        Use this when other output formats are desired.
+        """
+        img = self.pil_image()
+
+        if "dpi" not in kwargs.keys():
+            kwargs["dpi"] = (self.xres, self.yres)
+
+        img.save(*args, **kwargs)
+
+    def pil_tobytes(self, *args, **kwargs):
+        """Convert to an image in memory using Pillow.
+
+        An intermediate PIL Image is created, and its "save" method is used
+        to store the image. See Pillow documentation to learn about the
+        meaning of possible positional or keyword parameters.
+        Use this when other output formats are desired.
+        """
+        bytes_out = io.BytesIO()
+        img = self.pil_image()
+
+        if "dpi" not in kwargs.keys():
+            kwargs["dpi"] = (self.xres, self.yres)
+
+        img.save(bytes_out, *args, **kwargs)
+        return bytes_out.getvalue()
+
+    def pixel(self, x, y):
+        """Get color tuple of pixel (x, y).
+        Last item is the alpha if Pixmap.alpha is true."""
+        if g_use_extra:
+            return extra.pixmap_pixel(self.this.m_internal, x, y)
+        if (0
+                or x < 0
+                or x >= self.this.m_internal.w
+                or y < 0
+                or y >= self.this.m_internal.h
+                ):
+            RAISEPY(MSG_PIXEL_OUTSIDE, PyExc_ValueError)
+        n = self.this.m_internal.n
+        stride = self.this.m_internal.stride
+        i = stride * y + n * x
+        ret = tuple( self.samples_mv[ i: i+n])
+        return ret
+
+    @property
+    def samples(self)->bytes:
+        mv = self.samples_mv
+        return bytes( mv)
+
+    @property
+    def samples_mv(self):
+        '''
+        Pixmap samples memoryview.
+        '''
+        # We remember the returned memoryview so that our `__del__()` can
+        # release it; otherwise accessing it after we have been destructed will
+        # fail, possibly crashing Python; this is #4155.
+        #
+        if self._samples_mv is None:
+            self._samples_mv = mupdf.fz_pixmap_samples_memoryview(self.this)
+        return self._samples_mv
+    
+    def _samples_mv_release(self):
+        if self._samples_mv:
+            self._samples_mv.release()
+
+    @property
+    def samples_ptr(self):
+        return mupdf.fz_pixmap_samples_int(self.this)
+
+    def save(self, filename, output=None, jpg_quality=95):
+        """Output as image in format determined by filename extension.
+
+        Args:
+            output: (str) only use to overrule filename extension. Default is PNG.
+                    Others are JPEG, JPG, PNM, PGM, PPM, PBM, PAM, PSD, PS.
+        """
+        valid_formats = {
+                "png": 1,
+                "pnm": 2,
+                "pgm": 2,
+                "ppm": 2,
+                "pbm": 2,
+                "pam": 3,
+                "psd": 5,
+                "ps": 6,
+                "jpg": 7,
+                "jpeg": 7,
+                }
+        
+        if type(filename) is str:
+            pass
+        elif hasattr(filename, "absolute"):
+            filename = str(filename)
+        elif hasattr(filename, "name"):
+            filename = filename.name
+        if output is None:
+            _, ext = os.path.splitext(filename)
+            output = ext[1:]
+
+        idx = valid_formats.get(output.lower(), None)
+        if idx is None:
+            raise ValueError(f"Image format {output} not in {tuple(valid_formats.keys())}")
+        if self.alpha and idx in (2, 6, 7):
+            raise ValueError("'%s' cannot have alpha" % output)
+        if self.colorspace and self.colorspace.n > 3 and idx in (1, 2, 4):
+            raise ValueError(f"unsupported colorspace for '{output}'")
+        if idx == 7:
+            self.set_dpi(self.xres, self.yres)
+        return self._writeIMG(filename, idx, jpg_quality)
+
+    def set_alpha(self, alphavalues=None, premultiply=1, opaque=None, matte=None):
+        """Set alpha channel to values contained in a byte array.
+        If omitted, set alphas to 255.
+
+        Args:
+            alphavalues: (bytes) with length (width * height) or 'None'.
+            premultiply: (bool, True) premultiply colors with alpha values.
+            opaque: (tuple, length colorspace.n) this color receives opacity 0.
+            matte: (tuple, length colorspace.n)) preblending background color.
+        """
+        pix = self.this
+        alpha = 0
+        m = 0
+        if pix.alpha() == 0:
+            raise ValueError( MSG_PIX_NOALPHA)
+        n = mupdf.fz_pixmap_colorants(pix)
+        w = mupdf.fz_pixmap_width(pix)
+        h = mupdf.fz_pixmap_height(pix)
+        balen = w * h * (n+1)
+        colors = [0, 0, 0, 0]   # make this color opaque
+        bgcolor = [0, 0, 0, 0]  # preblending background color
+        zero_out = 0
+        bground = 0
+        if opaque and isinstance(opaque, (list, tuple)) and len(opaque) == n:
+            for i in range(n):
+                colors[i] = opaque[i]
+            zero_out = 1
+        if matte and isinstance( matte, (tuple, list)) and len(matte) == n:
+            for i in range(n):
+                bgcolor[i] = matte[i]
+            bground = 1
+        data = bytes()
+        data_len = 0
+        if alphavalues:
+            #res = JM_BufferFromBytes(alphavalues)
+            #data_len, data = mupdf.fz_buffer_storage(res)
+            #if data_len < w * h:
+            #    THROWMSG("bad alpha values")
+            # fixme: don't seem to need to create an fz_buffer - can
+            # use  directly?
+            if isinstance(alphavalues, (bytes, bytearray)):
+                data = alphavalues
+                data_len = len(alphavalues)
+            else:
+                assert 0, f'unexpected type for alphavalues: {type(alphavalues)}'
+            if data_len < w * h:
+                raise ValueError( "bad alpha values")
+        if 1:
+            # Use C implementation for speed.
+            mupdf.Pixmap_set_alpha_helper(
+                    balen,
+                    n,
+                    data_len,
+                    zero_out,
+                    mupdf.python_buffer_data( data),
+                    pix.m_internal,
+                    premultiply,
+                    bground,
+                    colors,
+                    bgcolor,
+                    )
+        else:
+            i = k = j = 0
+            data_fix = 255
+            while i < balen:
+                alpha = data[k]
+                if zero_out:
+                    for j in range(i, i+n):
+                        if mupdf.fz_samples_get(pix, j) != colors[j - i]:
+                            data_fix = 255
+                            break
+                        else:
+                            data_fix = 0
+                if data_len:
+                    def fz_mul255( a, b):
+                        x = a * b + 128
+                        x += x // 256
+                        return x // 256
+
+                    if data_fix == 0:
+                        mupdf.fz_samples_set(pix, i+n, 0)
+                    else:
+                        mupdf.fz_samples_set(pix, i+n, alpha)
+                    if premultiply and not bground:
+                        for j in range(i, i+n):
+                            mupdf.fz_samples_set(pix, j, fz_mul255( mupdf.fz_samples_get(pix, j), alpha))
+                    elif bground:
+                        for j in range( i, i+n):
+                            m = bgcolor[j - i]
+                            mupdf.fz_samples_set(pix, j, fz_mul255( mupdf.fz_samples_get(pix, j) - m, alpha))
+                else:
+                    mupdf.fz_samples_set(pix, i+n, data_fix)
+                i += n+1
+                k += 1
+
+    def tobytes(self, output="png", jpg_quality=95):
+        '''
+        Convert to binary image stream of desired type.
+        '''
+        valid_formats = {
+                "png": 1,
+                "pnm": 2,
+                "pgm": 2,
+                "ppm": 2,
+                "pbm": 2,
+                "pam": 3,
+                "tga": 4,
+                "tpic": 4,
+                "psd": 5,
+                "ps": 6,
+                'jpg': 7,
+                'jpeg': 7,
+                }
+        idx = valid_formats.get(output.lower(), None)
+        if idx is None:
+            raise ValueError(f"Image format {output} not in {tuple(valid_formats.keys())}")
+        if self.alpha and idx in (2, 6, 7):
+            raise ValueError("'{output}' cannot have alpha")
+        if self.colorspace and self.colorspace.n > 3 and idx in (1, 2, 4):
+            raise ValueError(f"unsupported colorspace for '{output}'")
+        if idx == 7:
+            self.set_dpi(self.xres, self.yres)
+        barray = self._tobytes(idx, jpg_quality)
+        return barray
+
+    def set_dpi(self, xres, yres):
+        """Set resolution in both dimensions."""
+        pm = self.this
+        pm.m_internal.xres = xres
+        pm.m_internal.yres = yres
+
+    def set_origin(self, x, y):
+        """Set top-left coordinates."""
+        pm = self.this
+        pm.m_internal.x = x
+        pm.m_internal.y = y
+
+    def set_pixel(self, x, y, color):
+        """Set color of pixel (x, y)."""
+        if g_use_extra:
+            return extra.set_pixel(self.this.m_internal, x, y, color)
+        pm = self.this
+        if not _INRANGE(x, 0, pm.w() - 1) or not _INRANGE(y, 0, pm.h() - 1):
+            raise ValueError( MSG_PIXEL_OUTSIDE)
+        n = pm.n()
+        for j in range(n):
+            i = color[j]
+            if not _INRANGE(i, 0, 255):
+                raise ValueError( MSG_BAD_COLOR_SEQ)
+        stride = mupdf.fz_pixmap_stride( pm)
+        i = stride * y + n * x
+        if 0:
+            # Using a cached self._memory_view doesn't actually make much
+            # difference to speed.
+            if not self._memory_view:
+                self._memory_view = self.samples_mv
+            for j in range(n):
+                self._memory_view[i + j] = color[j]
+        else:
+            for j in range(n):
+                pm.fz_samples_set(i + j, color[j])
+
+    def set_rect(self, bbox, color):
+        """Set color of all pixels in bbox."""
+        pm = self.this
+        n = pm.n()
+        c = []
+        for j in range(n):
+            i = color[j]
+            if not _INRANGE(i, 0, 255):
+                raise ValueError( MSG_BAD_COLOR_SEQ)
+            c.append(i)
+        bbox = JM_irect_from_py(bbox)
+        i = JM_fill_pixmap_rect_with_color(pm, c, bbox)
+        rc = bool(i)
+        return rc
+
+    def shrink(self, factor):
+        """Divide width and height by 2**factor.
+        E.g. factor=1 shrinks to 25% of original size (in place)."""
+        if factor < 1:
+            message_warning("ignoring shrink factor < 1")
+            return
+        mupdf.fz_subsample_pixmap( self.this, factor)
+        # Pixmap has changed so clear our memory view.
+        self._memory_view = None
+        self._samples_mv_release()
+
+    @property
+    def size(self):
+        """Pixmap size."""
+        return  mupdf.fz_pixmap_size( self.this)
+
+    @property
+    def stride(self):
+        """Length of one image line (width * n)."""
+        return self.this.stride()
+
+    def tint_with(self, black, white):
+        """Tint colors with modifiers for black and white."""
+        if not self.colorspace or self.colorspace.n > 3:
+            message("warning: colorspace invalid for function")
+            return
+        return mupdf.fz_tint_pixmap( self.this, black, white)
+
+    @property
+    def w(self):
+        """The width."""
+        return mupdf.fz_pixmap_width(self.this)
+    
+    def warp(self, quad, width, height):
+        """Return pixmap from a warped quad."""
+        if not quad.is_convex: raise ValueError("quad must be convex")
+        q = JM_quad_from_py(quad)
+        points = [ q.ul, q.ur, q.lr, q.ll]
+        dst = mupdf.fz_warp_pixmap( self.this, points, width, height)
+        return Pixmap( dst)
+
+    @property
+    def x(self):
+        """x component of Pixmap origin."""
+        return mupdf.fz_pixmap_x(self.this)
+
+    @property
+    def xres(self):
+        """Resolution in x direction."""
+        return self.this.xres()
+
+    @property
+    def y(self):
+        """y component of Pixmap origin."""
+        return mupdf.fz_pixmap_y(self.this)
+
+    @property
+    def yres(self):
+        """Resolution in y direction."""
+        return self.this.yres()
+
+    width  = w
+    height = h
+    
+    def __del__(self):
+        if self._samples_mv:
+            self._samples_mv.release()
+
+
+del Point
+class Point:
+
+    def __abs__(self):
+        return math.sqrt(self.x * self.x + self.y * self.y)
+
+    def __add__(self, p):
+        if hasattr(p, "__float__"):
+            return Point(self.x + p, self.y + p)
+        if len(p) != 2:
+            raise ValueError("Point: bad seq len")
+        return Point(self.x + p[0], self.y + p[1])
+
+    def __bool__(self):
+        return not (max(self) == min(self) == 0)
+
+    def __eq__(self, p):
+        if not hasattr(p, "__len__"):
+            return False
+        return len(p) == 2 and not (self - p)
+
+    def __getitem__(self, i):
+        return (self.x, self.y)[i]
+
+    def __hash__(self):
+        return hash(tuple(self))
+
+    def __init__(self, *args, x=None, y=None):
+        '''
+        Point() - all zeros
+        Point(x, y)
+        Point(Point) - new copy
+        Point(sequence) - from 'sequence'
+
+        Explicit keyword args x, y override earlier settings if not None.
+        '''
+        if not args:
+            self.x = 0.0
+            self.y = 0.0
+        elif len(args) > 2:
+            raise ValueError("Point: bad seq len")
+        elif len(args) == 2:
+            self.x = float(args[0])
+            self.y = float(args[1])
+        elif len(args) == 1:
+            l = args[0]
+            if isinstance(l, (mupdf.FzPoint, mupdf.fz_point)):
+                self.x = l.x
+                self.y = l.y
+            else:
+                if not hasattr(l, "__getitem__"):
+                    raise ValueError("Point: bad args")
+                if len(l) != 2:
+                    raise ValueError("Point: bad seq len")
+                self.x = float(l[0])
+                self.y = float(l[1])
+        else:
+            raise ValueError("Point: bad seq len")
+        if x is not None:   self.x = x
+        if y is not None:   self.y = y
+
+    def __len__(self):
+        return 2
+
+    def __mul__(self, m):
+        if hasattr(m, "__float__"):
+            return Point(self.x * m, self.y * m)
+        if hasattr(m, "__getitem__") and len(m) == 2:
+            # dot product
+            return self.x * m[0] + self.y * m[1]
+        p = Point(self)
+        return p.transform(m)
+
+    def __neg__(self):
+        return Point(-self.x, -self.y)
+
+    def __nonzero__(self):
+        return not (max(self) == min(self) == 0)
+
+    def __pos__(self):
+        return Point(self)
+
+    def __repr__(self):
+        return "Point" + str(tuple(self))
+
+    def __setitem__(self, i, v):
+        v = float(v)
+        if   i == 0: self.x = v
+        elif i == 1: self.y = v
+        else:
+            raise IndexError("index out of range")
+        return None
+
+    def __sub__(self, p):
+        if hasattr(p, "__float__"):
+            return Point(self.x - p, self.y - p)
+        if len(p) != 2:
+            raise ValueError("Point: bad seq len")
+        return Point(self.x - p[0], self.y - p[1])
+
+    def __truediv__(self, m):
+        if hasattr(m, "__float__"):
+            return Point(self.x * 1./m, self.y * 1./m)
+        m1 = util_invert_matrix(m)[1]
+        if not m1:
+            raise ZeroDivisionError("matrix not invertible")
+        p = Point(self)
+        return p.transform(m1)
+
+    @property
+    def abs_unit(self):
+        """Unit vector with positive coordinates."""
+        s = self.x * self.x + self.y * self.y
+        if s < EPSILON:
+            return Point(0,0)
+        s = math.sqrt(s)
+        return Point(abs(self.x) / s, abs(self.y) / s)
+
+    def distance_to(self, *args):
+        """Return distance to rectangle or another point."""
+        if not len(args) > 0:
+            raise ValueError("at least one parameter must be given")
+
+        x = args[0]
+        if len(x) == 2:
+            x = Point(x)
+        elif len(x) == 4:
+            x = Rect(x)
+        else:
+            raise ValueError("arg1 must be point-like or rect-like")
+
+        if len(args) > 1:
+            unit = args[1]
+        else:
+            unit = "px"
+        u = {"px": (1.,1.), "in": (1.,72.), "cm": (2.54, 72.),
+             "mm": (25.4, 72.)}
+        f = u[unit][0] / u[unit][1]
+
+        if type(x) is Point:
+            return abs(self - x) * f
+
+        # from here on, x is a rectangle
+        # as a safeguard, make a finite copy of it
+        r = Rect(x.top_left, x.top_left)
+        r = r | x.bottom_right
+        if self in r:
+            return 0.0
+        if self.x > r.x1:
+            if self.y >= r.y1:
+                return self.distance_to(r.bottom_right, unit)
+            elif self.y <= r.y0:
+                return self.distance_to(r.top_right, unit)
+            else:
+                return (self.x - r.x1) * f
+        elif r.x0 <= self.x <= r.x1:
+            if self.y >= r.y1:
+                return (self.y - r.y1) * f
+            else:
+                return (r.y0 - self.y) * f
+        else:
+            if self.y >= r.y1:
+                return self.distance_to(r.bottom_left, unit)
+            elif self.y <= r.y0:
+                return self.distance_to(r.top_left, unit)
+            else:
+                return (r.x0 - self.x) * f
+
+    def transform(self, m):
+        """Replace point by its transformation with matrix-like m."""
+        if len(m) != 6:
+            raise ValueError("Matrix: bad seq len")
+        self.x, self.y = util_transform_point(self, m)
+        return self
+
+    @property
+    def unit(self):
+        """Unit vector of the point."""
+        s = self.x * self.x + self.y * self.y
+        if s < EPSILON:
+            return Point(0,0)
+        s = math.sqrt(s)
+        return Point(self.x / s, self.y / s)
+
+    __div__ = __truediv__
+    norm = __abs__
+
+
+class Quad:
+
+    def __abs__(self):
+        if self.is_empty:
+            return 0.0
+        return abs(self.ul - self.ur) * abs(self.ul - self.ll)
+
+    def __add__(self, q):
+        if hasattr(q, "__float__"):
+            return Quad(self.ul + q, self.ur + q, self.ll + q, self.lr + q)
+        if len(q) != 4:
+            raise ValueError("Quad: bad seq len")
+        return Quad(self.ul + q[0], self.ur + q[1], self.ll + q[2], self.lr + q[3])
+
+    def __bool__(self):
+        return not self.is_empty
+
+    def __contains__(self, x):
+        try:
+            l = x.__len__()
+        except Exception:
+            if g_exceptions_verbose > 1:    exception_info()
+            return False
+        if l == 2:
+            return util_point_in_quad(x, self)
+        if l != 4:
+            return False
+        if CheckRect(x):
+            if Rect(x).is_empty:
+                return True
+            return util_point_in_quad(x[:2], self) and util_point_in_quad(x[2:], self)
+        if CheckQuad(x):
+            for i in range(4):
+                if not util_point_in_quad(x[i], self):
+                    return False
+            return True
+        return False
+
+    def __eq__(self, quad):
+        if not hasattr(quad, "__len__"):
+            return False
+        return len(quad) == 4 and (
+            self.ul == quad[0] and
+            self.ur == quad[1] and
+            self.ll == quad[2] and
+            self.lr == quad[3]
+        )
+
+    def __getitem__(self, i):
+        return (self.ul, self.ur, self.ll, self.lr)[i]
+
+    def __hash__(self):
+        return hash(tuple(self))
+
+    def __init__(self, *args, ul=None, ur=None, ll=None, lr=None):
+        '''
+        Quad() - all zero points
+        Quad(ul, ur, ll, lr)
+        Quad(quad) - new copy
+        Quad(sequence) - from 'sequence'
+
+        Explicit keyword args ul, ur, ll, lr override earlier settings if not
+        None.
+    
+        '''
+        if not args:
+            self.ul = self.ur = self.ll = self.lr = Point()
+        elif len(args) > 4:
+            raise ValueError("Quad: bad seq len")
+        elif len(args) == 4:
+            self.ul, self.ur, self.ll, self.lr = map(Point, args)
+        elif len(args) == 1:
+            l = args[0]
+            if isinstance(l, mupdf.FzQuad):
+                self.this = l
+                self.ul, self.ur, self.ll, self.lr = Point(l.ul), Point(l.ur), Point(l.ll), Point(l.lr)
+            elif not hasattr(l, "__getitem__"):
+                raise ValueError("Quad: bad args")
+            elif len(l) != 4:
+                raise ValueError("Quad: bad seq len")
+            else:
+                self.ul, self.ur, self.ll, self.lr = map(Point, l)
+        else:
+            raise ValueError("Quad: bad args")
+        if ul is not None:  self.ul = Point(ul)
+        if ur is not None:  self.ur = Point(ur)
+        if ll is not None:  self.ll = Point(ll)
+        if lr is not None:  self.lr = Point(lr)
+
+    def __len__(self):
+        return 4
+
+    def __mul__(self, m):
+        q = Quad(self)
+        q = q.transform(m)
+        return q
+
+    def __neg__(self):
+        return Quad(-self.ul, -self.ur, -self.ll, -self.lr)
+
+    def __nonzero__(self):
+        return not self.is_empty
+
+    def __pos__(self):
+        return Quad(self)
+
+    def __repr__(self):
+        return "Quad" + str(tuple(self))
+
+    def __setitem__(self, i, v):
+        if   i == 0: self.ul = Point(v)
+        elif i == 1: self.ur = Point(v)
+        elif i == 2: self.ll = Point(v)
+        elif i == 3: self.lr = Point(v)
+        else:
+            raise IndexError("index out of range")
+        return None
+
+    def __sub__(self, q):
+        if hasattr(q, "__float__"):
+            return Quad(self.ul - q, self.ur - q, self.ll - q, self.lr - q)
+        if len(q) != 4:
+            raise ValueError("Quad: bad seq len")
+        return Quad(self.ul - q[0], self.ur - q[1], self.ll - q[2], self.lr - q[3])
+
+    def __truediv__(self, m):
+        if hasattr(m, "__float__"):
+            im = 1. / m
+        else:
+            im = util_invert_matrix(m)[1]
+            if not im:
+                raise ZeroDivisionError("Matrix not invertible")
+        q = Quad(self)
+        q = q.transform(im)
+        return q
+
+    @property
+    def is_convex(self):
+        """Check if quad is convex and not degenerate.
+
+        Notes:
+            Check that for the two diagonals, the other two corners are not
+            on the same side of the diagonal.
+        Returns:
+            True or False.
+        """
+        m = planish_line(self.ul, self.lr)  # puts this diagonal on x-axis
+        p1 = self.ll * m  # transform the
+        p2 = self.ur * m  # other two points
+        if p1.y * p2.y > 0:
+            return False
+        m = planish_line(self.ll, self.ur)  # puts other diagonal on x-axis
+        p1 = self.lr * m  # transform the
+        p2 = self.ul * m  # remaining points
+        if p1.y * p2.y > 0:
+            return False
+        return True
+
+    @property
+    def is_empty(self):
+        """Check whether all quad corners are on the same line.
+
+        This is the case if width or height is zero.
+        """
+        return self.width < EPSILON or self.height < EPSILON
+
+    @property
+    def is_infinite(self):
+        """Check whether this is the infinite quad."""
+        return self.rect.is_infinite
+
+    @property
+    def is_rectangular(self):
+        """Check if quad is rectangular.
+
+        Notes:
+            Some rotation matrix can thus transform it into a rectangle.
+            This is equivalent to three corners enclose 90 degrees.
+        Returns:
+            True or False.
+        """
+
+        sine = util_sine_between(self.ul, self.ur, self.lr)
+        if abs(sine - 1) > EPSILON:  # the sine of the angle
+            return False
+
+        sine = util_sine_between(self.ur, self.lr, self.ll)
+        if abs(sine - 1) > EPSILON:
+            return False
+
+        sine = util_sine_between(self.lr, self.ll, self.ul)
+        if abs(sine - 1) > EPSILON:
+            return False
+
+        return True
+
+    def morph(self, p, m):
+        """Morph the quad with matrix-like 'm' and point-like 'p'.
+
+        Return a new quad."""
+        if self.is_infinite:
+            return INFINITE_QUAD()
+        delta = Matrix(1, 1).pretranslate(p.x, p.y)
+        q = self * ~delta * m * delta
+        return q
+
+    @property
+    def rect(self):
+        r = Rect()
+        r.x0 = min(self.ul.x, self.ur.x, self.lr.x, self.ll.x)
+        r.y0 = min(self.ul.y, self.ur.y, self.lr.y, self.ll.y)
+        r.x1 = max(self.ul.x, self.ur.x, self.lr.x, self.ll.x)
+        r.y1 = max(self.ul.y, self.ur.y, self.lr.y, self.ll.y)
+        return r
+
+    def transform(self, m):
+        """Replace quad by its transformation with matrix m."""
+        if hasattr(m, "__float__"):
+            pass
+        elif len(m) != 6:
+            raise ValueError("Matrix: bad seq len")
+        self.ul *= m
+        self.ur *= m
+        self.ll *= m
+        self.lr *= m
+        return self
+
+    __div__ = __truediv__
+    width  = property(lambda self: max(abs(self.ul - self.ur), abs(self.ll - self.lr)))
+    height = property(lambda self: max(abs(self.ul - self.ll), abs(self.ur - self.lr)))
+
+
+class Rect:
+    
+    def __abs__(self):
+        if self.is_empty or self.is_infinite:
+            return 0.0
+        return (self.x1 - self.x0) * (self.y1 - self.y0)
+
+    def __add__(self, p):
+        if hasattr(p, "__float__"):
+            return Rect(self.x0 + p, self.y0 + p, self.x1 + p, self.y1 + p)
+        if len(p) != 4:
+            raise ValueError("Rect: bad seq len")
+        return Rect(self.x0 + p[0], self.y0 + p[1], self.x1 + p[2], self.y1 + p[3])
+
+    def __and__(self, x):
+        if not hasattr(x, "__len__"):
+            raise ValueError("bad operand 2")
+
+        r1 = Rect(x)
+        r = Rect(self)
+        return r.intersect(r1)
+
+    def __bool__(self):
+        return not (max(self) == min(self) == 0)
+
+    def __contains__(self, x):
+        if hasattr(x, "__float__"):
+            return x in tuple(self)
+        l = len(x)
+        if l == 2:
+            return util_is_point_in_rect(x, self)
+        if l == 4:
+            r = INFINITE_RECT()
+            try:
+                r = Rect(x)
+            except Exception:
+                if g_exceptions_verbose > 1:    exception_info()
+                r = Quad(x).rect
+            return (self.x0 <= r.x0 <= r.x1 <= self.x1 and
+                    self.y0 <= r.y0 <= r.y1 <= self.y1)
+        return False
+
+    def __eq__(self, rect):
+        if not hasattr(rect, "__len__"):
+            return False
+        return len(rect) == 4 and not (self - rect)
+
+    def __getitem__(self, i):
+        return (self.x0, self.y0, self.x1, self.y1)[i]
+
+    def __hash__(self):
+        return hash(tuple(self))
+
+    def __init__(self, *args, p0=None, p1=None, x0=None, y0=None, x1=None, y1=None):
+        """
+        Rect() - all zeros
+        Rect(x0, y0, x1, y1)
+        Rect(top-left, x1, y1)
+        Rect(x0, y0, bottom-right)
+        Rect(top-left, bottom-right)
+        Rect(Rect or IRect) - new copy
+        Rect(sequence) - from 'sequence'
+    
+        Explicit keyword args p0, p1, x0, y0, x1, y1 override earlier settings
+        if not None.
+        """
+        x0, y0, x1, y1 = util_make_rect( *args, p0=p0, p1=p1, x0=x0, y0=y0, x1=x1, y1=y1)
+        self.x0 = float( x0)
+        self.y0 = float( y0)
+        self.x1 = float( x1)
+        self.y1 = float( y1)
+
+    def __len__(self):
+        return 4
+
+    def __mul__(self, m):
+        if hasattr(m, "__float__"):
+            return Rect(self.x0 * m, self.y0 * m, self.x1 * m, self.y1 * m)
+        r = Rect(self)
+        r = r.transform(m)
+        return r
+
+    def __neg__(self):
+        return Rect(-self.x0, -self.y0, -self.x1, -self.y1)
+
+    def __nonzero__(self):
+        return not (max(self) == min(self) == 0)
+
+    def __or__(self, x):
+        if not hasattr(x, "__len__"):
+            raise ValueError("bad operand 2")
+        r = Rect(self)
+        if len(x) == 2:
+            return r.include_point(x)
+        if len(x) == 4:
+            return r.include_rect(x)
+        raise ValueError("bad operand 2")
+
+    def __pos__(self):
+        return Rect(self)
+
+    def __repr__(self):
+        return "Rect" + str(tuple(self))
+
+    def __setitem__(self, i, v):
+        v = float(v)
+        if   i == 0: self.x0 = v
+        elif i == 1: self.y0 = v
+        elif i == 2: self.x1 = v
+        elif i == 3: self.y1 = v
+        else:
+            raise IndexError("index out of range")
+        return None
+
+    def __sub__(self, p):
+        if hasattr(p, "__float__"):
+            return Rect(self.x0 - p, self.y0 - p, self.x1 - p, self.y1 - p)
+        if len(p) != 4:
+            raise ValueError("Rect: bad seq len")
+        return Rect(self.x0 - p[0], self.y0 - p[1], self.x1 - p[2], self.y1 - p[3])
+
+    def __truediv__(self, m):
+        if hasattr(m, "__float__"):
+            return Rect(self.x0 * 1./m, self.y0 * 1./m, self.x1 * 1./m, self.y1 * 1./m)
+        im = util_invert_matrix(m)[1]
+        if not im:
+            raise ZeroDivisionError(f"Matrix not invertible: {m}")
+        r = Rect(self)
+        r = r.transform(im)
+        return r
+
+    @property
+    def bottom_left(self):
+        """Bottom-left corner."""
+        return Point(self.x0, self.y1)
+
+    @property
+    def bottom_right(self):
+        """Bottom-right corner."""
+        return Point(self.x1, self.y1)
+
+    def contains(self, x):
+        """Check if containing point-like or rect-like x."""
+        return self.__contains__(x)
+
+    @property
+    def height(self):
+        return max(0, self.y1 - self.y0)
+
+    def include_point(self, p):
+        """Extend to include point-like p."""
+        if len(p) != 2:
+            raise ValueError("Point: bad seq len")
+        self.x0, self.y0, self.x1, self.y1 = util_include_point_in_rect(self, p)
+        return self
+
+    def include_rect(self, r):
+        """Extend to include rect-like r."""
+        if len(r) != 4:
+            raise ValueError("Rect: bad seq len")
+        r = Rect(r)
+        if r.is_infinite or self.is_infinite:
+            self.x0, self.y0, self.x1, self.y1 = FZ_MIN_INF_RECT, FZ_MIN_INF_RECT, FZ_MAX_INF_RECT, FZ_MAX_INF_RECT
+        elif r.is_empty:
+            return self
+        elif self.is_empty:
+            self.x0, self.y0, self.x1, self.y1 = r.x0, r.y0, r.x1, r.y1
+        else:
+            self.x0, self.y0, self.x1, self.y1 = util_union_rect(self, r)
+        return self
+
+    def intersect(self, r):
+        """Restrict to common rect with rect-like r."""
+        if not len(r) == 4:
+            raise ValueError("Rect: bad seq len")
+        r = Rect(r)
+        if r.is_infinite:
+            return self
+        elif self.is_infinite:
+            self.x0, self.y0, self.x1, self.y1 = r.x0, r.y0, r.x1, r.y1
+        elif r.is_empty:
+            self.x0, self.y0, self.x1, self.y1 = r.x0, r.y0, r.x1, r.y1
+        elif self.is_empty:
+            return self
+        else:
+            self.x0, self.y0, self.x1, self.y1 = util_intersect_rect(self, r)
+        return self
+
+    def intersects(self, x):
+        """Check if intersection with rectangle x is not empty."""
+        r1 = Rect(x)
+        if self.is_empty or self.is_infinite or r1.is_empty or r1.is_infinite:
+            return False
+        r = Rect(self)
+        if r.intersect(r1).is_empty:
+            return False
+        return True
+
+    @property
+    def is_empty(self):
+        """True if rectangle area is empty."""
+        return self.x0 >= self.x1 or self.y0 >= self.y1
+
+    @property
+    def is_infinite(self):
+        """True if this is the infinite rectangle."""
+        return self.x0 == self.y0 == FZ_MIN_INF_RECT and self.x1 == self.y1 == FZ_MAX_INF_RECT
+
+    @property
+    def is_valid(self):
+        """True if rectangle is valid."""
+        return self.x0 <= self.x1 and self.y0 <= self.y1
+
+    def morph(self, p, m):
+        """Morph with matrix-like m and point-like p.
+
+        Returns a new quad."""
+        if self.is_infinite:
+            return INFINITE_QUAD()
+        return self.quad.morph(p, m)
+
+    def norm(self):
+        return math.sqrt(sum([c*c for c in self]))
+
+    def normalize(self):
+        """Replace rectangle with its finite version."""
+        if self.x1 < self.x0:
+            self.x0, self.x1 = self.x1, self.x0
+        if self.y1 < self.y0:
+            self.y0, self.y1 = self.y1, self.y0
+        return self
+
+    @property
+    def quad(self):
+        """Return Quad version of rectangle."""
+        return Quad(self.tl, self.tr, self.bl, self.br)
+
+    def round(self):
+        """Return the IRect."""
+        return IRect(util_round_rect(self))
+
+    @property
+    def top_left(self):
+        """Top-left corner."""
+        return Point(self.x0, self.y0)
+
+    @property
+    def top_right(self):
+        """Top-right corner."""
+        return Point(self.x1, self.y0)
+    
+    def torect(self, r):
+        """Return matrix that converts to target rect."""
+
+        r = Rect(r)
+        if self.is_infinite or self.is_empty or r.is_infinite or r.is_empty:
+            raise ValueError("rectangles must be finite and not empty")
+        return (
+            Matrix(1, 0, 0, 1, -self.x0, -self.y0)
+            * Matrix(r.width / self.width, r.height / self.height)
+            * Matrix(1, 0, 0, 1, r.x0, r.y0)
+        )
+
+    def transform(self, m):
+        """Replace with the transformation by matrix-like m."""
+        if not len(m) == 6:
+            raise ValueError("Matrix: bad seq len")
+        self.x0, self.y0, self.x1, self.y1 = util_transform_rect(self, m)
+        return self
+
+    @property
+    def width(self):
+        return max(0, self.x1 - self.x0)
+
+    __div__ = __truediv__
+
+    bl = bottom_left
+    br = bottom_right
+    irect = property(round)
+    tl = top_left
+    tr = top_right
+
+
+class Story:
+
+    def __init__( self, html='', user_css=None, em=12, archive=None):
+        buffer_ = mupdf.fz_new_buffer_from_copied_data( html.encode('utf-8'))
+        if archive and not isinstance(archive, Archive):
+            archive = Archive(archive)
+        arch = archive.this if archive else mupdf.FzArchive( None)
+        if hasattr(mupdf, 'FzStoryS'):
+            self.this = mupdf.FzStoryS( buffer_, user_css, em, arch)
+        else:
+            self.this = mupdf.FzStory( buffer_, user_css, em, arch)
+    
+    def add_header_ids(self):
+        '''
+        Look for `` items in `self` and adds unique `id`
+        attributes if not already present.
+        '''
+        dom = self.body
+        i = 0
+        x = dom.find(None, None, None)
+        while x:
+            name = x.tagname
+            if len(name) == 2 and name[0]=="h" and name[1] in "123456":
+                attr = x.get_attribute_value("id")
+                if not attr:
+                    id_ = f"h_id_{i}"
+                    #log(f"{name=}: setting {id_=}")
+                    x.set_attribute("id", id_)
+                    i += 1
+            x = x.find_next(None, None, None)
+
+    @staticmethod
+    def add_pdf_links(document_or_stream, positions):
+        """
+        Adds links to PDF document.
+        Args:
+            document_or_stream:
+                A PDF `Document` or raw PDF content, for example an
+                `io.BytesIO` instance.
+            positions:
+                List of `ElementPosition`'s for `document_or_stream`,
+                typically from Story.element_positions(). We raise an
+                exception if two or more positions have same id.
+        Returns:
+            `document_or_stream` if a `Document` instance, otherwise a
+            new `Document` instance.
+        We raise an exception if an `href` in `positions` refers to an
+        internal position `#` but no item in `positions` has `id =
+        name`.
+        """
+        if isinstance(document_or_stream, Document):
+            document = document_or_stream
+        else:
+            document = Document("pdf", document_or_stream)
+
+        # Create dict from id to position, which we will use to find
+        # link destinations.
+        #
+        id_to_position = dict()
+        #log(f"positions: {positions}")
+        for position in positions:
+            #log(f"add_pdf_links(): position: {position}")
+            if (position.open_close & 1) and position.id:
+                #log(f"add_pdf_links(): position with id: {position}")
+                if position.id in id_to_position:
+                    #log(f"Ignoring duplicate positions with id={position.id!r}")
+                    pass
+                else:
+                    id_to_position[ position.id] = position
+
+        # Insert links for all positions that have an `href`.
+        #
+        for position_from in positions:
+        
+            if (position_from.open_close & 1) and position_from.href:
+            
+                #log(f"add_pdf_links(): position with href: {position}")
+                link = dict()
+                link['from'] = Rect(position_from.rect)
+                
+                if position_from.href.startswith("#"):
+                    #`...` internal link.
+                    target_id = position_from.href[1:]
+                    try:
+                        position_to = id_to_position[ target_id]
+                    except Exception as e:
+                        if g_exceptions_verbose > 1:    exception_info()
+                        raise RuntimeError(f"No destination with id={target_id}, required by position_from: {position_from}") from e
+                    # Make link from `position_from`'s rect to top-left of
+                    # `position_to`'s rect.
+                    if 0:
+                        log(f"add_pdf_links(): making link from:")
+                        log(f"add_pdf_links():    {position_from}")
+                        log(f"add_pdf_links(): to:")
+                        log(f"add_pdf_links():    {position_to}")
+                    link["kind"] = LINK_GOTO
+                    x0, y0, x1, y1 = position_to.rect
+                    # This appears to work well with viewers which scroll
+                    # to make destination point top-left of window.
+                    link["to"] = Point(x0, y0)
+                    link["page"] = position_to.page_num - 1
+                    
+                else:
+                    # `...` external link.
+                    if position_from.href.startswith('name:'):
+                        link['kind'] = LINK_NAMED
+                        link['name'] = position_from.href[5:]
+                    else:
+                        link['kind'] = LINK_URI
+                        link['uri'] = position_from.href
+                
+                #log(f'Adding link: {position_from.page_num=} {link=}.')
+                document[position_from.page_num - 1].insert_link(link)
+        
+        return document
+
+    @property
+    def body(self):
+        dom = self.document()
+        return dom.bodytag()
+        
+    def document( self):
+        dom = mupdf.fz_story_document( self.this)
+        return Xml( dom)
+
+    def draw( self, device, matrix=None):
+        ctm2 = JM_matrix_from_py( matrix)
+        dev = device.this if device else mupdf.FzDevice( None)
+        mupdf.fz_draw_story( self.this, dev, ctm2)
+
+    def element_positions( self, function, args=None):
+        '''
+        Trigger a callback function to record where items have been placed.
+        '''
+        if type(args) is dict:
+            for k in args.keys():
+                if not (type(k) is str and k.isidentifier()):
+                    raise ValueError(f"invalid key '{k}'")
+        else:
+            args = {}
+        if not callable(function) or function.__code__.co_argcount != 1:
+            raise ValueError("callback 'function' must be a callable with exactly one argument")
+        
+        def function2( position):
+            class Position2:
+                pass
+            position2 = Position2()
+            position2.depth = position.depth
+            position2.heading = position.heading
+            position2.id = position.id
+            position2.rect = JM_py_from_rect(position.rect)
+            position2.text = position.text
+            position2.open_close = position.open_close
+            position2.rect_num = position.rectangle_num
+            position2.href = position.href
+            if args:
+                for k, v in args.items():
+                    setattr( position2, k, v)
+            function( position2)
+        mupdf.fz_story_positions( self.this, function2)
+
+    def place( self, where):
+        where = JM_rect_from_py( where)
+        filled = mupdf.FzRect()
+        more = mupdf.fz_place_story( self.this, where, filled)
+        return more, JM_py_from_rect( filled)
+
+    def reset( self):
+        mupdf.fz_reset_story( self.this)
+    
+    def write(self, writer, rectfn, positionfn=None, pagefn=None):
+        dev = None
+        page_num = 0
+        rect_num = 0
+        filled = Rect(0, 0, 0, 0)
+        while 1:
+            mediabox, rect, ctm = rectfn(rect_num, filled)
+            rect_num += 1
+            if mediabox:
+                # new page.
+                page_num += 1
+            more, filled = self.place( rect)
+            if positionfn:
+                def positionfn2(position):
+                    # We add a `.page_num` member to the
+                    # `ElementPosition` instance.
+                    position.page_num = page_num
+                    positionfn(position)
+                self.element_positions(positionfn2)
+            if writer:
+                if mediabox:
+                    # new page.
+                    if dev:
+                        if pagefn:
+                            pagefn(page_num, mediabox, dev, 1)
+                        writer.end_page()
+                    dev = writer.begin_page( mediabox)
+                    if pagefn:
+                        pagefn(page_num, mediabox, dev, 0)
+                self.draw( dev, ctm)
+                if not more:
+                    if pagefn:
+                        pagefn( page_num, mediabox, dev, 1)
+                    writer.end_page()
+            else:
+                self.draw(None, ctm)
+            if not more:
+                break
+
+    @staticmethod
+    def write_stabilized(writer, contentfn, rectfn, user_css=None, em=12, positionfn=None, pagefn=None, archive=None, add_header_ids=True):
+        positions = list()
+        content = None
+        # Iterate until stable.
+        while 1:
+            content_prev = content
+            content = contentfn( positions)
+            stable = False
+            if content == content_prev:
+                stable = True
+            content2 = content
+            story = Story(content2, user_css, em, archive)
+
+            if add_header_ids:
+                story.add_header_ids()
+
+            positions = list()
+            def positionfn2(position):
+                #log(f"write_stabilized(): {stable=} {positionfn=} {position=}")
+                positions.append(position)
+                if stable and positionfn:
+                    positionfn(position)
+            story.write(
+                    writer if stable else None,
+                    rectfn,
+                    positionfn2,
+                    pagefn,
+                    )
+            if stable:
+                break
+
+    @staticmethod
+    def write_stabilized_with_links(contentfn, rectfn, user_css=None, em=12, positionfn=None, pagefn=None, archive=None, add_header_ids=True):
+        #log("write_stabilized_with_links()")
+        stream = io.BytesIO()
+        writer = DocumentWriter(stream)
+        positions = []
+        def positionfn2(position):
+            #log(f"write_stabilized_with_links(): {position=}")
+            positions.append(position)
+            if positionfn:
+                positionfn(position)
+        Story.write_stabilized(writer, contentfn, rectfn, user_css, em, positionfn2, pagefn, archive, add_header_ids)
+        writer.close()
+        stream.seek(0)
+        return Story.add_pdf_links(stream, positions)
+
+    def write_with_links(self, rectfn, positionfn=None, pagefn=None):
+        #log("write_with_links()")
+        stream = io.BytesIO()
+        writer = DocumentWriter(stream)
+        positions = []
+        def positionfn2(position):
+            #log(f"write_with_links(): {position=}")
+            positions.append(position)
+            if positionfn:
+                positionfn(position)
+        self.write(writer, rectfn, positionfn=positionfn2, pagefn=pagefn)
+        writer.close()
+        stream.seek(0)
+        return Story.add_pdf_links(stream, positions)
+
+    class FitResult:
+        '''
+        The result from a `Story.fit*()` method.
+        
+        Members:
+        
+        `big_enough`:
+            `True` if the fit succeeded.
+        `filled`:
+            From the last call to `Story.place()`.
+        `more`:
+            `False` if the fit succeeded.
+        `numcalls`:
+            Number of calls made to `self.place()`.
+        `parameter`:
+            The successful parameter value, or the largest failing value.
+        `rect`:
+            The rect created from `parameter`.
+        '''
+        def __init__(self, big_enough=None, filled=None, more=None, numcalls=None, parameter=None, rect=None):
+            self.big_enough = big_enough
+            self.filled = filled
+            self.more = more
+            self.numcalls = numcalls
+            self.parameter = parameter
+            self.rect = rect
+        
+        def __repr__(self):
+            return (
+                    f' big_enough={self.big_enough}'
+                    f' filled={self.filled}'
+                    f' more={self.more}'
+                    f' numcalls={self.numcalls}'
+                    f' parameter={self.parameter}'
+                    f' rect={self.rect}'
+                    )
+
+    def fit(self, fn, pmin=None, pmax=None, delta=0.001, verbose=False):
+        '''
+        Finds optimal rect that contains the story `self`.
+        
+        Returns a `Story.FitResult` instance.
+            
+        On success, the last call to `self.place()` will have been with the
+        returned rectangle, so `self.draw()` can be used directly.
+        
+        Args:
+        :arg fn:
+            A callable taking a floating point `parameter` and returning a
+            `pymupdf.Rect()`. If the rect is empty, we assume the story will
+            not fit and do not call `self.place()`.
+
+            Must guarantee that `self.place()` behaves monotonically when
+            given rect `fn(parameter`) as `parameter` increases. This
+            usually means that both width and height increase or stay
+            unchanged as `parameter` increases.
+        :arg pmin:
+            Minimum parameter to consider; `None` for -infinity.
+        :arg pmax:
+            Maximum parameter to consider; `None` for +infinity.
+        :arg delta:
+            Maximum error in returned `parameter`.
+        :arg verbose:
+            If true we output diagnostics.
+        '''
+        def log(text):
+            assert verbose
+            message(f'fit(): {text}')
+        
+        assert isinstance(pmin, (int, float)) or pmin is None
+        assert isinstance(pmax, (int, float)) or pmax is None
+        
+        class State:
+            def __init__(self):
+                self.pmin = pmin
+                self.pmax = pmax
+                self.pmin_result = None
+                self.pmax_result = None
+                self.result = None
+                self.numcalls = 0
+                if verbose:
+                    self.pmin0 = pmin
+                    self.pmax0 = pmax
+        state = State()
+        
+        if verbose:
+            log(f'starting. {state.pmin=} {state.pmax=}.')
+        
+        self.reset()
+
+        def ret():
+            if state.pmax is not None:
+                if state.last_p != state.pmax:
+                    if verbose:
+                        log(f'Calling update() with pmax, because was overwritten by later calls.')
+                    big_enough = update(state.pmax)
+                    assert big_enough
+                result = state.pmax_result
+            else:
+                result = state.pmin_result if state.pmin_result else Story.FitResult(numcalls=state.numcalls)
+            if verbose:
+                log(f'finished. {state.pmin0=} {state.pmax0=} {state.pmax=}: returning {result=}')
+            return result
+        
+        def update(parameter):
+            '''
+            Evaluates `more, _ = self.place(fn(parameter))`. If `more` is
+            false, then `rect` is big enough to contain `self` and we
+            set `state.pmax=parameter` and return True. Otherwise we set
+            `state.pmin=parameter` and return False.
+            '''
+            rect = fn(parameter)
+            assert isinstance(rect, Rect), f'{type(rect)=} {rect=}'
+            if rect.is_empty:
+                big_enough = False
+                result = Story.FitResult(parameter=parameter, numcalls=state.numcalls)
+                if verbose:
+                    log(f'update(): not calling self.place() because rect is empty.')
+            else:
+                more, filled = self.place(rect)
+                state.numcalls += 1
+                big_enough = not more
+                result = Story.FitResult(
+                        filled=filled,
+                        more=more,
+                        numcalls=state.numcalls,
+                        parameter=parameter,
+                        rect=rect,
+                        big_enough=big_enough,
+                        )
+                if verbose:
+                    log(f'update(): called self.place(): {state.numcalls:>2d}: {more=} {parameter=} {rect=}.')
+            if big_enough:
+                state.pmax = parameter
+                state.pmax_result = result
+            else:
+                state.pmin = parameter
+                state.pmin_result = result
+            state.last_p = parameter
+            return big_enough
+
+        def opposite(p, direction):
+            '''
+            Returns same sign as `direction`, larger or smaller than `p` if
+            direction is positive or negative respectively.
+            '''
+            if p is None or p==0:
+                return direction
+            if direction * p > 0:
+                return 2 * p
+            return -p
+            
+        if state.pmin is None:
+            # Find an initial finite pmin value.
+            if verbose: log(f'finding pmin.')
+            parameter = opposite(state.pmax, -1)
+            while 1:
+                if not update(parameter):
+                    break
+                parameter *= 2
+        else:
+            if update(state.pmin):
+                if verbose: log(f'{state.pmin=} is big enough.')
+                return ret()
+        
+        if state.pmax is None:
+            # Find an initial finite pmax value.
+            if verbose: log(f'finding pmax.')
+            parameter = opposite(state.pmin, +1)
+            while 1:
+                if update(parameter):
+                    break
+                parameter *= 2
+        else:
+            if not update(state.pmax):
+                # No solution possible.
+                state.pmax = None
+                if verbose: log(f'No solution possible {state.pmax=}.')
+                return ret()
+        
+        # Do binary search in pmin..pmax.
+        if verbose: log(f'doing binary search with {state.pmin=} {state.pmax=}.')
+        while 1:
+            if state.pmax - state.pmin < delta:
+                return ret()
+            parameter = (state.pmin + state.pmax) / 2
+            update(parameter)
+
+    def fit_scale(self, rect, scale_min=0, scale_max=None, delta=0.001, verbose=False):
+        '''
+        Finds smallest value `scale` in range `scale_min..scale_max` where
+        `scale * rect` is large enough to contain the story `self`.
+
+        Returns a `Story.FitResult` instance.
+
+        :arg width:
+            width of rect.
+        :arg height:
+            height of rect.
+        :arg scale_min:
+            Minimum scale to consider; must be >= 0.
+        :arg scale_max:
+            Maximum scale to consider, must be >= scale_min or `None` for
+            infinite.
+        :arg delta:
+            Maximum error in returned scale.
+        :arg verbose:
+            If true we output diagnostics.
+        '''
+        x0, y0, x1, y1 = rect
+        width = x1 - x0
+        height = y1 - y0
+        def fn(scale):
+            return Rect(x0, y0, x0 + scale*width, y0 + scale*height)
+        return self.fit(fn, scale_min, scale_max, delta, verbose)
+
+    def fit_height(self, width, height_min=0, height_max=None, origin=(0, 0), delta=0.001, verbose=False):
+        '''
+        Finds smallest height in range `height_min..height_max` where a rect
+        with size `(width, height)` is large enough to contain the story
+        `self`.
+
+        Returns a `Story.FitResult` instance.
+
+        :arg width:
+            width of rect.
+        :arg height_min:
+            Minimum height to consider; must be >= 0.
+        :arg height_max:
+            Maximum height to consider, must be >= height_min or `None` for
+            infinite.
+        :arg origin:
+            `(x0, y0)` of rect.
+        :arg delta:
+            Maximum error in returned height.
+        :arg verbose:
+            If true we output diagnostics.
+        '''
+        x0, y0 = origin
+        x1 = x0 + width
+        def fn(height):
+            return Rect(x0, y0, x1, y0+height)
+        return self.fit(fn, height_min, height_max, delta, verbose)
+
+    def fit_width(self, height, width_min=0, width_max=None, origin=(0, 0), delta=0.001, verbose=False):
+        '''
+        Finds smallest width in range `width_min..width_max` where a rect with size
+        `(width, height)` is large enough to contain the story `self`.
+
+        Returns a `Story.FitResult` instance.
+        Returns a `FitResult` instance.
+
+        :arg height:
+            height of rect.
+        :arg width_min:
+            Minimum width to consider; must be >= 0.
+        :arg width_max:
+            Maximum width to consider, must be >= width_min or `None` for
+            infinite.
+        :arg origin:
+            `(x0, y0)` of rect.
+        :arg delta:
+            Maximum error in returned width.
+        :arg verbose:
+            If true we output diagnostics.
+        '''
+        x0, y0 = origin
+        y1 = y0 + height
+        def fn(width):
+            return Rect(x0, y0, x0+width, y1)
+        return self.fit(fn, width_min, width_max, delta, verbose)
+
+
+class TextPage:
+
+    def __init__(self, *args):
+        if args_match(args, mupdf.FzRect):
+            mediabox = args[0]
+            self.this = mupdf.FzStextPage( mediabox)
+        elif args_match(args, mupdf.FzStextPage):
+            self.this = args[0]
+        else:
+            raise Exception(f'Unrecognised args: {args}')
+        self.thisown = True
+        self.parent = None
+
+    def _extractText(self, format_):
+        this_tpage = self.this
+        res = mupdf.fz_new_buffer(1024)
+        out = mupdf.FzOutput( res)
+        # fixme: mupdfwrap.py thinks fz_output is not copyable, possibly
+        # because there is no .refs member visible and no fz_keep_output() fn,
+        # although there is an fz_drop_output(). So mupdf.fz_new_output_with_buffer()
+        # doesn't convert the returned fz_output* into a mupdf.FzOutput.
+        #out = mupdf.FzOutput(out)
+        if format_ == 1:
+            mupdf.fz_print_stext_page_as_html(out, this_tpage, 0)
+        elif format_ == 3:
+            mupdf.fz_print_stext_page_as_xml(out, this_tpage, 0)
+        elif format_ == 4:
+            mupdf.fz_print_stext_page_as_xhtml(out, this_tpage, 0)
+        else:
+            JM_print_stext_page_as_text(res, this_tpage)
+        out.fz_close_output()
+        text = JM_EscapeStrFromBuffer(res)
+        return text
+
+    def _getNewBlockList(self, page_dict, raw):
+        JM_make_textpage_dict(self.this, page_dict, raw)
+
+    def _textpage_dict(self, raw=False):
+        page_dict = {"width": self.rect.width, "height": self.rect.height}
+        self._getNewBlockList(page_dict, raw)
+        return page_dict
+
+    def extractBLOCKS(self):
+        """Return a list with text block information."""
+        if g_use_extra:
+            return extra.extractBLOCKS(self.this)
+        block_n = -1
+        this_tpage = self.this
+        tp_rect = mupdf.FzRect(this_tpage.m_internal.mediabox)
+        res = mupdf.fz_new_buffer(1024)
+        lines = []
+        for block in this_tpage:
+            block_n += 1
+            blockrect = mupdf.FzRect(mupdf.FzRect.Fixed_EMPTY)
+            if block.m_internal.type == mupdf.FZ_STEXT_BLOCK_TEXT:
+                mupdf.fz_clear_buffer(res) # set text buffer to empty
+                line_n = -1
+                last_char = 0
+                for line in block:
+                    line_n += 1
+                    linerect = mupdf.FzRect(mupdf.FzRect.Fixed_EMPTY)
+                    for ch in line:
+                        cbbox = JM_char_bbox(line, ch)
+                        if (not JM_rects_overlap(tp_rect, cbbox)
+                                and not mupdf.fz_is_infinite_rect(tp_rect)
+                                ):
+                            continue
+                        JM_append_rune(res, ch.m_internal.c)
+                        last_char = ch.m_internal.c
+                        linerect = mupdf.fz_union_rect(linerect, cbbox)
+                    if last_char != 10 and not mupdf.fz_is_empty_rect(linerect):
+                        mupdf.fz_append_byte(res, 10)
+                    blockrect = mupdf.fz_union_rect(blockrect, linerect)
+                text = JM_EscapeStrFromBuffer(res)
+            elif (JM_rects_overlap(tp_rect, block.m_internal.bbox)
+                    or mupdf.fz_is_infinite_rect(tp_rect)
+                    ):
+                img = block.i_image()
+                cs = img.colorspace()
+                text = "" % (
+                        mupdf.fz_colorspace_name(cs),
+                        img.w(), img.h(), img.bpc()
+                        )
+                blockrect = mupdf.fz_union_rect(blockrect, mupdf.FzRect(block.m_internal.bbox))
+            if not mupdf.fz_is_empty_rect(blockrect):
+                litem = (
+                        blockrect.x0,
+                        blockrect.y0,
+                        blockrect.x1,
+                        blockrect.y1,
+                        text,
+                        block_n,
+                        block.m_internal.type,
+                        )
+                lines.append(litem)
+        return lines
+
+    def extractDICT(self, cb=None, sort=False) -> dict:
+        """Return page content as a Python dict of images and text spans."""
+        val = self._textpage_dict(raw=False)
+        if cb is not None:
+            val["width"] = cb.width
+            val["height"] = cb.height
+        if sort:
+            blocks = val["blocks"]
+            blocks.sort(key=lambda b: (b["bbox"][3], b["bbox"][0]))
+            val["blocks"] = blocks
+        return val
+
+    def extractHTML(self) -> str:
+        """Return page content as a HTML string."""
+        return self._extractText(1)
+
+    def extractIMGINFO(self, hashes=0):
+        """Return a list with image meta information."""
+        block_n = -1
+        this_tpage = self.this
+        rc = []
+        for block in this_tpage:
+            block_n += 1
+            if block.m_internal.type == mupdf.FZ_STEXT_BLOCK_TEXT:
+                continue
+            img = block.i_image()
+            img_size = 0
+            mask = img.mask()
+            if mask.m_internal:
+                has_mask = True
+            else:
+                has_mask = False
+            compr_buff = mupdf.fz_compressed_image_buffer(img)
+            if compr_buff.m_internal:
+                img_size = compr_buff.fz_compressed_buffer_size()
+                compr_buff = None
+            if hashes:
+                r = mupdf.FzIrect(FZ_MIN_INF_RECT, FZ_MIN_INF_RECT, FZ_MAX_INF_RECT, FZ_MAX_INF_RECT)
+                assert mupdf.fz_is_infinite_irect(r)
+                m = mupdf.FzMatrix(img.w(), 0, 0, img.h(), 0, 0)
+                pix, w, h = mupdf.fz_get_pixmap_from_image(img, r, m)
+                digest = mupdf.fz_md5_pixmap2(pix)
+                digest = bytes(digest)
+                if img_size == 0:
+                    img_size = img.w() * img.h() * img.n()
+            cs = mupdf.FzColorspace(mupdf.ll_fz_keep_colorspace(img.m_internal.colorspace))
+            block_dict = dict()
+            block_dict[dictkey_number] = block_n
+            block_dict[dictkey_bbox] = JM_py_from_rect(block.m_internal.bbox)
+            block_dict[dictkey_matrix] = JM_py_from_matrix(block.i_transform())
+            block_dict[dictkey_width] = img.w()
+            block_dict[dictkey_height] = img.h()
+            block_dict[dictkey_colorspace] = mupdf.fz_colorspace_n(cs)
+            block_dict[dictkey_cs_name] = mupdf.fz_colorspace_name(cs)
+            block_dict[dictkey_xres] = img.xres()
+            block_dict[dictkey_yres] = img.yres()
+            block_dict[dictkey_bpc] = img.bpc()
+            block_dict[dictkey_size] = img_size
+            if hashes:
+                block_dict["digest"] = digest
+            block_dict["has-mask"] = has_mask
+            rc.append(block_dict)
+        return rc
+
+    def extractJSON(self, cb=None, sort=False) -> str:
+        """Return 'extractDICT' converted to JSON format."""
+        import base64
+        import json
+        val = self._textpage_dict(raw=False)
+
+        class b64encode(json.JSONEncoder):
+            def default(self, s):
+                if type(s) in (bytes, bytearray):
+                    return base64.b64encode(s).decode()
+
+        if cb is not None:
+            val["width"] = cb.width
+            val["height"] = cb.height
+        if sort:
+            blocks = val["blocks"]
+            blocks.sort(key=lambda b: (b["bbox"][3], b["bbox"][0]))
+            val["blocks"] = blocks
+        
+        val = json.dumps(val, separators=(",", ":"), cls=b64encode, indent=1)
+        return val
+
+    def extractRAWDICT(self, cb=None, sort=False) -> dict:
+        """Return page content as a Python dict of images and text characters."""
+        val = self._textpage_dict(raw=True)
+        if cb is not None:
+            val["width"] = cb.width
+            val["height"] = cb.height
+        if sort:
+            blocks = val["blocks"]
+            blocks.sort(key=lambda b: (b["bbox"][3], b["bbox"][0]))
+            val["blocks"] = blocks
+        return val
+
+    def extractRAWJSON(self, cb=None, sort=False) -> str:
+        """Return 'extractRAWDICT' converted to JSON format."""
+        import base64
+        import json
+        val = self._textpage_dict(raw=True)
+
+        class b64encode(json.JSONEncoder):
+            def default(self,s):
+                if type(s) in (bytes, bytearray):
+                    return base64.b64encode(s).decode()
+
+        if cb is not None:
+            val["width"] = cb.width
+            val["height"] = cb.height
+        if sort:
+            blocks = val["blocks"]
+            blocks.sort(key=lambda b: (b["bbox"][3], b["bbox"][0]))
+            val["blocks"] = blocks
+        val = json.dumps(val, separators=(",", ":"), cls=b64encode, indent=1)
+        return val
+
+    def extractSelection(self, pointa, pointb):
+        a = JM_point_from_py(pointa)
+        b = JM_point_from_py(pointb)
+        found = mupdf.fz_copy_selection(self.this, a, b, 0)
+        return found
+
+    def extractText(self, sort=False) -> str:
+        """Return simple, bare text on the page."""
+        if not sort:
+            return self._extractText(0)
+        blocks = self.extractBLOCKS()[:]
+        blocks.sort(key=lambda b: (b[3], b[0]))
+        return "".join([b[4] for b in blocks])
+
+    def extractTextbox(self, rect):
+        this_tpage = self.this
+        assert isinstance(this_tpage, mupdf.FzStextPage)
+        area = JM_rect_from_py(rect)
+        found = JM_copy_rectangle(this_tpage, area)
+        rc = PyUnicode_DecodeRawUnicodeEscape(found)
+        return rc
+
+    def extractWORDS(self, delimiters=None):
+        """Return a list with text word information."""
+        if g_use_extra:
+            return extra.extractWORDS(self.this, delimiters)
+        buflen = 0
+        last_char_rtl = 0
+        block_n = -1
+        wbbox = mupdf.FzRect(mupdf.FzRect.Fixed_EMPTY)  # word bbox
+        this_tpage = self.this
+        tp_rect = mupdf.FzRect(this_tpage.m_internal.mediabox)
+
+        lines = None
+        buff = mupdf.fz_new_buffer(64)
+        lines = []
+        for block in this_tpage:
+            block_n += 1
+            if block.m_internal.type != mupdf.FZ_STEXT_BLOCK_TEXT:
+                continue
+            line_n = -1
+            for line in block:
+                line_n += 1
+                word_n = 0                  # word counter per line
+                mupdf.fz_clear_buffer(buff) # reset word buffer
+                buflen = 0                  # reset char counter
+                for ch in line:
+                    cbbox = JM_char_bbox(line, ch)
+                    if (not JM_rects_overlap(tp_rect, cbbox)
+                            and not mupdf.fz_is_infinite_rect(tp_rect)
+                            ):
+                        continue
+                    word_delimiter = JM_is_word_delimiter(ch.m_internal.c, delimiters)
+                    this_char_rtl = JM_is_rtl_char(ch.m_internal.c)
+                    if word_delimiter or this_char_rtl != last_char_rtl:
+                        if buflen == 0 and word_delimiter:
+                            continue    # skip delimiters at line start
+                        if not mupdf.fz_is_empty_rect(wbbox):
+                            word_n, wbbox = JM_append_word(lines, buff, wbbox, block_n, line_n, word_n)
+                        mupdf.fz_clear_buffer(buff)
+                        buflen = 0  # reset char counter
+                        if word_delimiter:
+                            continue
+                    # append one unicode character to the word
+                    JM_append_rune(buff, ch.m_internal.c)
+                    last_char_rtl = this_char_rtl
+                    buflen += 1
+                    # enlarge word bbox
+                    wbbox = mupdf.fz_union_rect(wbbox, JM_char_bbox(line, ch))
+                if buflen and not mupdf.fz_is_empty_rect(wbbox):
+                    word_n, wbbox = JM_append_word(lines, buff, wbbox, block_n, line_n, word_n)
+                buflen = 0
+        return lines
+
+    def extractXHTML(self) -> str:
+        """Return page content as a XHTML string."""
+        return self._extractText(4)
+
+    def extractXML(self) -> str:
+        """Return page content as a XML string."""
+        return self._extractText(3)
+
+    def poolsize(self):
+        """TextPage current poolsize."""
+        tpage = self.this
+        pool = mupdf.Pool(tpage.m_internal.pool)
+        size = mupdf.fz_pool_size( pool)
+        pool.m_internal = None  # Ensure that pool's destructor does not free the pool.
+        return size
+
+    @property
+    def rect(self):
+        """Page rectangle."""
+        this_tpage = self.this
+        mediabox = this_tpage.m_internal.mediabox
+        val = JM_py_from_rect(mediabox)
+        val = Rect(val)
+
+        return val
+
+    def search(self, needle, hit_max=0, quads=1):
+        """Locate 'needle' returning rects or quads."""
+        val = JM_search_stext_page(self.this, needle)
+        if not val:
+            return val
+        items = len(val)
+        for i in range(items):  # change entries to quads or rects
+            q = Quad(val[i])
+            if quads:
+                val[i] = q
+            else:
+                val[i] = q.rect
+        if quads:
+            return val
+        i = 0  # join overlapping rects on the same line
+        while i < items - 1:
+            v1 = val[i]
+            v2 = val[i + 1]
+            if v1.y1 != v2.y1 or (v1 & v2).is_empty:
+                i += 1
+                continue  # no overlap on same line
+            val[i] = v1 | v2  # join rectangles
+            del val[i + 1]  # remove v2
+            items -= 1  # reduce item count
+        return val
+
+    extractTEXT = extractText
+
+
+class TextWriter:
+
+    def __init__(self, page_rect, opacity=1, color=None):
+        """Stores text spans for later output on compatible PDF pages."""
+        self.this = mupdf.fz_new_text()
+
+        self.opacity = opacity
+        self.color = color
+        self.rect = Rect(page_rect)
+        self.ctm = Matrix(1, 0, 0, -1, 0, self.rect.height)
+        self.ictm = ~self.ctm
+        self.last_point = Point()
+        self.last_point.__doc__ = "Position following last text insertion."
+        self.text_rect = Rect()
+        
+        self.text_rect.__doc__ = "Accumulated area of text spans."
+        self.used_fonts = set()
+        self.thisown = True
+
+    @property
+    def _bbox(self):
+        val = JM_py_from_rect( mupdf.fz_bound_text( self.this, mupdf.FzStrokeState(None), mupdf.FzMatrix()))
+        val = Rect(val)
+        return val
+
+    def append(self, pos, text, font=None, fontsize=11, language=None, right_to_left=0, small_caps=0):
+        """Store 'text' at point 'pos' using 'font' and 'fontsize'."""
+        pos = Point(pos) * self.ictm
+        #log( '{font=}')
+        if font is None:
+            font = Font("helv")
+        if not font.is_writable:
+            if 0:
+                log( '{font.this.m_internal.name=}')
+                log( '{font.this.m_internal.t3matrix=}')
+                log( '{font.this.m_internal.bbox=}')
+                log( '{font.this.m_internal.glyph_count=}')
+                log( '{font.this.m_internal.use_glyph_bbox=}')
+                log( '{font.this.m_internal.width_count=}')
+                log( '{font.this.m_internal.width_default=}')
+                log( '{font.this.m_internal.has_digest=}')
+                log( 'Unsupported font {font.name=}')
+                if mupdf_cppyy:
+                    import cppyy
+                    log( f'Unsupported font {cppyy.gbl.mupdf_font_name(font.this.m_internal)=}')
+            raise ValueError("Unsupported font '%s'." % font.name)
+        if right_to_left:
+            text = self.clean_rtl(text)
+            text = "".join(reversed(text))
+            right_to_left = 0
+
+        lang = mupdf.fz_text_language_from_string(language)
+        p = JM_point_from_py(pos)
+        trm = mupdf.fz_make_matrix(fontsize, 0, 0, fontsize, p.x, p.y)
+        markup_dir = 0
+        wmode = 0
+        if small_caps == 0:
+            trm = mupdf.fz_show_string( self.this, font.this, trm, text, wmode, right_to_left, markup_dir, lang)
+        else:
+            trm = JM_show_string_cs( self.this, font.this, trm, text, wmode, right_to_left, markup_dir, lang)
+        val = JM_py_from_matrix(trm)
+
+        self.last_point = Point(val[-2:]) * self.ctm
+        self.text_rect = self._bbox * self.ctm
+        val = self.text_rect, self.last_point
+        if font.flags["mono"] == 1:
+            self.used_fonts.add(font)
+        return val
+
+    def appendv(self, pos, text, font=None, fontsize=11, language=None, small_caps=False):
+        lheight = fontsize * 1.2
+        for c in text:
+            self.append(pos, c, font=font, fontsize=fontsize,
+                language=language, small_caps=small_caps)
+            pos.y += lheight
+        return self.text_rect, self.last_point
+
+    def clean_rtl(self, text):
+        """Revert the sequence of Latin text parts.
+
+        Text with right-to-left writing direction (Arabic, Hebrew) often
+        contains Latin parts, which are written in left-to-right: numbers, names,
+        etc. For output as PDF text we need *everything* in right-to-left.
+        E.g. an input like " ABCDE FG HIJ  KL " will be
+        converted to " JIH GF EDCBA  LK ". The Arabic
+        parts remain untouched.
+
+        Args:
+            text: str
+        Returns:
+            Massaged string.
+        """
+        if not text:
+            return text
+        # split into words at space boundaries
+        words = text.split(" ")
+        idx = []
+        for i in range(len(words)):
+            w = words[i]
+        # revert character sequence for Latin only words
+            if not (len(w) < 2 or max([ord(c) for c in w]) > 255):
+                words[i] = "".join(reversed(w))
+                idx.append(i)  # stored index of Latin word
+
+        # adjacent Latin words must revert their sequence, too
+        idx2 = []  # store indices of adjacent Latin words
+        for i in range(len(idx)):
+            if idx2 == []:  # empty yet?
+                idx2.append(idx[i]) # store Latin word number
+
+            elif idx[i] > idx2[-1] + 1:  # large gap to last?
+                if len(idx2) > 1:  # at least two consecutives?
+                    words[idx2[0] : idx2[-1] + 1] = reversed(
+                        words[idx2[0] : idx2[-1] + 1]
+                    )  # revert their sequence
+                idx2 = [idx[i]]  # re-initialize
+
+            elif idx[i] == idx2[-1] + 1:  # new adjacent Latin word
+                idx2.append(idx[i])
+
+        text = " ".join(words)
+        return text
+
+    def write_text(self, page, color=None, opacity=-1, overlay=1, morph=None, matrix=None, render_mode=0, oc=0):
+        """Write the text to a PDF page having the TextWriter's page size.
+
+        Args:
+            page: a PDF page having same size.
+            color: override text color.
+            opacity: override transparency.
+            overlay: put in foreground or background.
+            morph: tuple(Point, Matrix), apply a matrix with a fixpoint.
+            matrix: Matrix to be used instead of 'morph' argument.
+            render_mode: (int) PDF render mode operator 'Tr'.
+        """
+        CheckParent(page)
+        if abs(self.rect - page.rect) > 1e-3:
+            raise ValueError("incompatible page rect")
+        if morph is not None:
+            if (type(morph) not in (tuple, list)
+                    or type(morph[0]) is not Point
+                    or type(morph[1]) is not Matrix
+                    ):
+                raise ValueError("morph must be (Point, Matrix) or None")
+        if matrix is not None and morph is not None:
+            raise ValueError("only one of matrix, morph is allowed")
+        if getattr(opacity, "__float__", None) is None or opacity == -1:
+            opacity = self.opacity
+        if color is None:
+            color = self.color
+
+        if 1:
+            pdfpage = page._pdf_page()
+            alpha = 1
+            if opacity >= 0 and opacity < 1:
+                alpha = opacity
+            ncol = 1
+            dev_color = [0, 0, 0, 0]
+            if color:
+                ncol, dev_color = JM_color_FromSequence(color)
+            if ncol == 3:
+                colorspace = mupdf.fz_device_rgb()
+            elif ncol == 4:
+                colorspace = mupdf.fz_device_cmyk()
+            else:
+                colorspace = mupdf.fz_device_gray()
+
+            resources = mupdf.pdf_new_dict(pdfpage.doc(), 5)
+            contents = mupdf.fz_new_buffer(1024)
+            dev = mupdf.pdf_new_pdf_device( pdfpage.doc(), mupdf.FzMatrix(), resources, contents)
+            #log( '=== {dev_color!r=}')
+            mupdf.fz_fill_text(
+                    dev,
+                    self.this,
+                    mupdf.FzMatrix(),
+                    colorspace,
+                    dev_color,
+                    alpha,
+                    mupdf.FzColorParams(mupdf.fz_default_color_params),
+                    )
+            mupdf.fz_close_device( dev)
+
+            # copy generated resources into the one of the page
+            max_nums = JM_merge_resources( pdfpage, resources)
+            cont_string = JM_EscapeStrFromBuffer( contents)
+            result = (max_nums, cont_string)
+            val = result
+
+        max_nums = val[0]
+        content = val[1]
+        max_alp, max_font = max_nums
+        old_cont_lines = content.splitlines()
+
+        optcont = page._get_optional_content(oc)
+        if optcont is not None:
+            bdc = "/OC /%s BDC" % optcont
+            emc = "EMC"
+        else:
+            bdc = emc = ""
+
+        new_cont_lines = ["q"]
+        if bdc:
+            new_cont_lines.append(bdc)
+
+        cb = page.cropbox_position
+        if page.rotation in (90, 270):
+            delta = page.rect.height - page.rect.width
+        else:
+            delta = 0
+        mb = page.mediabox
+        if bool(cb) or mb.y0 != 0 or delta != 0:
+            new_cont_lines.append(f"1 0 0 1 {_format_g((cb.x, cb.y + mb.y0 - delta))} cm")
+
+        if morph:
+            p = morph[0] * self.ictm
+            delta = Matrix(1, 1).pretranslate(p.x, p.y)
+            matrix = ~delta * morph[1] * delta
+        if morph or matrix:
+            new_cont_lines.append(_format_g(JM_TUPLE(matrix)) + " cm")
+
+        for line in old_cont_lines:
+            if line.endswith(" cm"):
+                continue
+            if line == "BT":
+                new_cont_lines.append(line)
+                new_cont_lines.append("%i Tr" % render_mode)
+                continue
+            if line.endswith(" gs"):
+                alp = int(line.split()[0][4:]) + max_alp
+                line = "/Alp%i gs" % alp
+            elif line.endswith(" Tf"):
+                temp = line.split()
+                fsize = float(temp[1])
+                if render_mode != 0:
+                    w = fsize * 0.05
+                else:
+                    w = 1
+                new_cont_lines.append(_format_g(w) + " w")
+                font = int(temp[0][2:]) + max_font
+                line = " ".join(["/F%i" % font] + temp[1:])
+            elif line.endswith(" rg"):
+                new_cont_lines.append(line.replace("rg", "RG"))
+            elif line.endswith(" g"):
+                new_cont_lines.append(line.replace(" g", " G"))
+            elif line.endswith(" k"):
+                new_cont_lines.append(line.replace(" k", " K"))
+            new_cont_lines.append(line)
+        if emc:
+            new_cont_lines.append(emc)
+        new_cont_lines.append("Q\n")
+        content = "\n".join(new_cont_lines).encode("utf-8")
+        TOOLS._insert_contents(page, content, overlay=overlay)
+        val = None
+        for font in self.used_fonts:
+            repair_mono_font(page, font)
+        return val
+
+
+class IRect:
+    """
+    IRect() - all zeros
+    IRect(x0, y0, x1, y1) - 4 coordinates
+    IRect(top-left, x1, y1) - point and 2 coordinates
+    IRect(x0, y0, bottom-right) - 2 coordinates and point
+    IRect(top-left, bottom-right) - 2 points
+    IRect(sequ) - new from sequence or rect-like
+    """
+
+    def __add__(self, p):
+        return Rect.__add__(self, p).round()
+
+    def __and__(self, x):
+        return Rect.__and__(self, x).round()
+
+    def __contains__(self, x):
+        return Rect.__contains__(self, x)
+
+    def __eq__(self, r):
+        if not hasattr(r, "__len__"):
+            return False
+        return len(r) == 4 and self.x0 == r[0] and self.y0 == r[1] and self.x1 == r[2] and self.y1 == r[3]
+
+    def __getitem__(self, i):
+        return (self.x0, self.y0, self.x1, self.y1)[i]
+
+    def __hash__(self):
+        return hash(tuple(self))
+
+    def __init__(self, *args, p0=None, p1=None, x0=None, y0=None, x1=None, y1=None):
+        self.x0, self.y0, self.x1, self.y1 = util_make_irect( *args, p0=p0, p1=p1, x0=x0, y0=y0, x1=x1, y1=y1)
+
+    def __len__(self):
+        return 4
+
+    def __mul__(self, m):
+        return Rect.__mul__(self, m).round()
+
+    def __neg__(self):
+        return IRect(-self.x0, -self.y0, -self.x1, -self.y1)
+
+    def __or__(self, x):
+        return Rect.__or__(self, x).round()
+
+    def __pos__(self):
+        return IRect(self)
+
+    def __repr__(self):
+        return "IRect" + str(tuple(self))
+
+    def __setitem__(self, i, v):
+        v = int(v)
+        if   i == 0: self.x0 = v
+        elif i == 1: self.y0 = v
+        elif i == 2: self.x1 = v
+        elif i == 3: self.y1 = v
+        else:
+            raise IndexError("index out of range")
+        return None
+
+    def __sub__(self, p):
+        return Rect.__sub__(self, p).round()
+
+    def __truediv__(self, m):
+        return Rect.__truediv__(self, m).round()
+
+    @property
+    def bottom_left(self):
+        """Bottom-left corner."""
+        return Point(self.x0, self.y1)
+
+    @property
+    def bottom_right(self):
+        """Bottom-right corner."""
+        return Point(self.x1, self.y1)
+
+    @property
+    def height(self):
+        return max(0, self.y1 - self.y0)
+
+    def include_point(self, p):
+        """Extend rectangle to include point p."""
+        rect = self.rect.include_point(p)
+        return rect.irect
+
+    def include_rect(self, r):
+        """Extend rectangle to include rectangle r."""
+        rect = self.rect.include_rect(r)
+        return rect.irect
+
+    def intersect(self, r):
+        """Restrict rectangle to intersection with rectangle r."""
+        return Rect.intersect(self, r).round()
+
+    def intersects(self, x):
+        return Rect.intersects(self, x)
+
+    @property
+    def is_empty(self):
+        """True if rectangle area is empty."""
+        return self.x0 >= self.x1 or self.y0 >= self.y1
+
+    @property
+    def is_infinite(self):
+        """True if rectangle is infinite."""
+        return self.x0 == self.y0 == FZ_MIN_INF_RECT and self.x1 == self.y1 == FZ_MAX_INF_RECT
+
+    @property
+    def is_valid(self):
+        """True if rectangle is valid."""
+        return self.x0 <= self.x1 and self.y0 <= self.y1
+
+    def morph(self, p, m):
+        """Morph with matrix-like m and point-like p.
+
+        Returns a new quad."""
+        if self.is_infinite:
+            return INFINITE_QUAD()
+        return self.quad.morph(p, m)
+
+    def norm(self):
+        return math.sqrt(sum([c*c for c in self]))
+
+    def normalize(self):
+        """Replace rectangle with its valid version."""
+        if self.x1 < self.x0:
+            self.x0, self.x1 = self.x1, self.x0
+        if self.y1 < self.y0:
+            self.y0, self.y1 = self.y1, self.y0
+        return self
+
+    @property
+    def quad(self):
+        """Return Quad version of rectangle."""
+        return Quad(self.tl, self.tr, self.bl, self.br)
+
+    @property
+    def rect(self):
+        return Rect(self)
+
+    @property
+    def top_left(self):
+        """Top-left corner."""
+        return Point(self.x0, self.y0)
+
+    @property
+    def top_right(self):
+        """Top-right corner."""
+        return Point(self.x1, self.y0)
+
+    def torect(self, r):
+        """Return matrix that converts to target rect."""
+        r = Rect(r)
+        if self.is_infinite or self.is_empty or r.is_infinite or r.is_empty:
+            raise ValueError("rectangles must be finite and not empty")
+        return (
+                Matrix(1, 0, 0, 1, -self.x0, -self.y0)
+                * Matrix(r.width / self.width, r.height / self.height)
+                * Matrix(1, 0, 0, 1, r.x0, r.y0)
+                )
+
+    def transform(self, m):
+        return Rect.transform(self, m).round()
+
+    @property
+    def width(self):
+        return max(0, self.x1 - self.x0)
+
+    br = bottom_right
+    bl = bottom_left
+    tl = top_left
+    tr = top_right
+
+
+# Data
+#
+
+if 1:
+    _self = sys.modules[__name__]
+    if 1:
+        for _name, _value in mupdf.__dict__.items():
+            if _name.startswith(('PDF_', 'UCDN_SCRIPT_')):
+                if _name.startswith('PDF_ENUM_NAME_'):
+                    # Not a simple enum.
+                    pass
+                else:
+                    #assert not inspect.isroutine(value)
+                    #log(f'importing {_name=} {_value=}.')
+                    setattr(_self, _name, _value)
+                    #log(f'{getattr( self, name, None)=}')
+    else:
+        # This is slow due to importing inspect, e.g. 0.019 instead of 0.004.
+        for _name, _value in inspect.getmembers(mupdf):
+            if _name.startswith(('PDF_', 'UCDN_SCRIPT_')):
+                if _name.startswith('PDF_ENUM_NAME_'):
+                    # Not a simple enum.
+                    pass
+                else:
+                    #assert not inspect.isroutine(value)
+                    #log(f'importing {name}')
+                    setattr(_self, _name, _value)
+                    #log(f'{getattr( self, name, None)=}')
+    
+    # This is a macro so not preserved in mupdf C++/Python bindings.
+    #
+    PDF_SIGNATURE_DEFAULT_APPEARANCE = (0
+            | mupdf.PDF_SIGNATURE_SHOW_LABELS
+            | mupdf.PDF_SIGNATURE_SHOW_DN
+            | mupdf.PDF_SIGNATURE_SHOW_DATE
+            | mupdf.PDF_SIGNATURE_SHOW_TEXT_NAME
+            | mupdf.PDF_SIGNATURE_SHOW_GRAPHIC_NAME
+            | mupdf.PDF_SIGNATURE_SHOW_LOGO
+            )
+
+    #UCDN_SCRIPT_ADLAM = mupdf.UCDN_SCRIPT_ADLAM
+    #setattr(self, 'UCDN_SCRIPT_ADLAM', mupdf.UCDN_SCRIPT_ADLAM)
+    
+    assert mupdf.UCDN_EAST_ASIAN_H == 1
+    
+    # Flake8 incorrectly fails next two lines because we've dynamically added
+    # items to self.
+    assert PDF_TX_FIELD_IS_MULTILINE == mupdf.PDF_TX_FIELD_IS_MULTILINE # noqa: F821
+    assert UCDN_SCRIPT_ADLAM == mupdf.UCDN_SCRIPT_ADLAM # noqa: F821
+    del _self, _name, _value
+
+AnyType = typing.Any
+
+Base14_fontnames = (
+    "Courier",
+    "Courier-Oblique",
+    "Courier-Bold",
+    "Courier-BoldOblique",
+    "Helvetica",
+    "Helvetica-Oblique",
+    "Helvetica-Bold",
+    "Helvetica-BoldOblique",
+    "Times-Roman",
+    "Times-Italic",
+    "Times-Bold",
+    "Times-BoldItalic",
+    "Symbol",
+    "ZapfDingbats",
+    )
+
+Base14_fontdict = {}
+for f in Base14_fontnames:
+    Base14_fontdict[f.lower()] = f
+Base14_fontdict["helv"] = "Helvetica"
+Base14_fontdict["heit"] = "Helvetica-Oblique"
+Base14_fontdict["hebo"] = "Helvetica-Bold"
+Base14_fontdict["hebi"] = "Helvetica-BoldOblique"
+Base14_fontdict["cour"] = "Courier"
+Base14_fontdict["coit"] = "Courier-Oblique"
+Base14_fontdict["cobo"] = "Courier-Bold"
+Base14_fontdict["cobi"] = "Courier-BoldOblique"
+Base14_fontdict["tiro"] = "Times-Roman"
+Base14_fontdict["tibo"] = "Times-Bold"
+Base14_fontdict["tiit"] = "Times-Italic"
+Base14_fontdict["tibi"] = "Times-BoldItalic"
+Base14_fontdict["symb"] = "Symbol"
+Base14_fontdict["zadb"] = "ZapfDingbats"
+
+EPSILON = 1e-5
+FLT_EPSILON = 1e-5
+
+# largest 32bit integers surviving C float conversion roundtrips
+# used by MuPDF to define infinite rectangles
+FZ_MIN_INF_RECT = -0x80000000
+FZ_MAX_INF_RECT = 0x7fffff80
+
+JM_annot_id_stem = "fitz"
+JM_mupdf_warnings_store = []
+JM_mupdf_show_errors = 1
+JM_mupdf_show_warnings = 0
+
+
+# ------------------------------------------------------------------------------
+# Image recompression constants
+# ------------------------------------------------------------------------------
+FZ_RECOMPRESS_NEVER = mupdf.FZ_RECOMPRESS_NEVER
+FZ_RECOMPRESS_SAME = mupdf.FZ_RECOMPRESS_SAME
+FZ_RECOMPRESS_LOSSLESS = mupdf.FZ_RECOMPRESS_LOSSLESS
+FZ_RECOMPRESS_JPEG = mupdf.FZ_RECOMPRESS_JPEG
+FZ_RECOMPRESS_J2K = mupdf.FZ_RECOMPRESS_J2K
+FZ_RECOMPRESS_FAX = mupdf.FZ_RECOMPRESS_FAX
+FZ_SUBSAMPLE_AVERAGE = mupdf.FZ_SUBSAMPLE_AVERAGE
+FZ_SUBSAMPLE_BICUBIC = mupdf.FZ_SUBSAMPLE_BICUBIC
+
+# ------------------------------------------------------------------------------
+# Various PDF Optional Content Flags
+# ------------------------------------------------------------------------------
+PDF_OC_ON = 0
+PDF_OC_TOGGLE = 1
+PDF_OC_OFF = 2
+
+# ------------------------------------------------------------------------------
+# link kinds and link flags
+# ------------------------------------------------------------------------------
+LINK_NONE = 0
+LINK_GOTO = 1
+LINK_URI = 2
+LINK_LAUNCH = 3
+LINK_NAMED = 4
+LINK_GOTOR = 5
+LINK_FLAG_L_VALID = 1
+LINK_FLAG_T_VALID = 2
+LINK_FLAG_R_VALID = 4
+LINK_FLAG_B_VALID = 8
+LINK_FLAG_FIT_H = 16
+LINK_FLAG_FIT_V = 32
+LINK_FLAG_R_IS_ZOOM = 64
+
+SigFlag_SignaturesExist = 1
+SigFlag_AppendOnly = 2
+
+STAMP_Approved = 0
+STAMP_AsIs = 1
+STAMP_Confidential = 2
+STAMP_Departmental = 3
+STAMP_Experimental = 4
+STAMP_Expired = 5
+STAMP_Final = 6
+STAMP_ForComment = 7
+STAMP_ForPublicRelease = 8
+STAMP_NotApproved = 9
+STAMP_NotForPublicRelease = 10
+STAMP_Sold = 11
+STAMP_TopSecret = 12
+STAMP_Draft = 13
+
+TEXT_ALIGN_LEFT = 0
+TEXT_ALIGN_CENTER = 1
+TEXT_ALIGN_RIGHT = 2
+TEXT_ALIGN_JUSTIFY = 3
+
+TEXT_FONT_SUPERSCRIPT = 1
+TEXT_FONT_ITALIC = 2
+TEXT_FONT_SERIFED = 4
+TEXT_FONT_MONOSPACED = 8
+TEXT_FONT_BOLD = 16
+
+TEXT_OUTPUT_TEXT = 0
+TEXT_OUTPUT_HTML = 1
+TEXT_OUTPUT_JSON = 2
+TEXT_OUTPUT_XML = 3
+TEXT_OUTPUT_XHTML = 4
+
+TEXT_PRESERVE_LIGATURES = mupdf.FZ_STEXT_PRESERVE_LIGATURES
+TEXT_PRESERVE_WHITESPACE = mupdf.FZ_STEXT_PRESERVE_WHITESPACE
+TEXT_PRESERVE_IMAGES = mupdf.FZ_STEXT_PRESERVE_IMAGES
+TEXT_INHIBIT_SPACES = mupdf.FZ_STEXT_INHIBIT_SPACES
+TEXT_DEHYPHENATE = mupdf.FZ_STEXT_DEHYPHENATE
+TEXT_PRESERVE_SPANS = mupdf.FZ_STEXT_PRESERVE_SPANS
+TEXT_MEDIABOX_CLIP = mupdf.FZ_STEXT_MEDIABOX_CLIP
+TEXT_USE_CID_FOR_UNKNOWN_UNICODE = mupdf.FZ_STEXT_USE_CID_FOR_UNKNOWN_UNICODE
+TEXT_COLLECT_STRUCTURE = mupdf.FZ_STEXT_COLLECT_STRUCTURE
+TEXT_ACCURATE_BBOXES = mupdf.FZ_STEXT_ACCURATE_BBOXES
+TEXT_COLLECT_VECTORS = mupdf.FZ_STEXT_COLLECT_VECTORS
+TEXT_IGNORE_ACTUALTEXT = mupdf.FZ_STEXT_IGNORE_ACTUALTEXT
+TEXT_SEGMENT = mupdf.FZ_STEXT_SEGMENT
+
+if mupdf_version_tuple >= (1, 26):
+    TEXT_PARAGRAPH_BREAK = mupdf.FZ_STEXT_PARAGRAPH_BREAK
+    TEXT_TABLE_HUNT = mupdf.FZ_STEXT_TABLE_HUNT
+    TEXT_COLLECT_STYLES = mupdf.FZ_STEXT_COLLECT_STYLES
+    TEXT_USE_GID_FOR_UNKNOWN_UNICODE = mupdf.FZ_STEXT_USE_GID_FOR_UNKNOWN_UNICODE
+    TEXT_CLIP_RECT = mupdf.FZ_STEXT_CLIP_RECT
+    TEXT_ACCURATE_ASCENDERS = mupdf.FZ_STEXT_ACCURATE_ASCENDERS
+    TEXT_ACCURATE_SIDE_BEARINGS = mupdf.FZ_STEXT_ACCURATE_SIDE_BEARINGS
+
+# 2025-05-07: Non-standard names preserved for backwards compatibility.
+TEXT_STEXT_SEGMENT = TEXT_SEGMENT
+TEXT_CID_FOR_UNKNOWN_UNICODE = TEXT_USE_CID_FOR_UNKNOWN_UNICODE
+
+TEXTFLAGS_WORDS = (0
+        | TEXT_PRESERVE_LIGATURES
+        | TEXT_PRESERVE_WHITESPACE
+        | TEXT_MEDIABOX_CLIP
+        | TEXT_USE_CID_FOR_UNKNOWN_UNICODE
+        )
+
+TEXTFLAGS_BLOCKS = (0
+        | TEXT_PRESERVE_LIGATURES
+        | TEXT_PRESERVE_WHITESPACE
+        | TEXT_MEDIABOX_CLIP
+        | TEXT_USE_CID_FOR_UNKNOWN_UNICODE
+        )
+
+TEXTFLAGS_DICT = (0
+        | TEXT_PRESERVE_LIGATURES
+        | TEXT_PRESERVE_WHITESPACE
+        | TEXT_MEDIABOX_CLIP
+        | TEXT_PRESERVE_IMAGES
+        | TEXT_USE_CID_FOR_UNKNOWN_UNICODE
+        )
+
+TEXTFLAGS_RAWDICT = TEXTFLAGS_DICT
+
+TEXTFLAGS_SEARCH = (0
+        | TEXT_PRESERVE_WHITESPACE
+        | TEXT_MEDIABOX_CLIP
+        | TEXT_DEHYPHENATE
+        | TEXT_USE_CID_FOR_UNKNOWN_UNICODE
+        )
+
+TEXTFLAGS_HTML = (0
+        | TEXT_PRESERVE_LIGATURES
+        | TEXT_PRESERVE_WHITESPACE
+        | TEXT_MEDIABOX_CLIP
+        | TEXT_PRESERVE_IMAGES
+        | TEXT_USE_CID_FOR_UNKNOWN_UNICODE
+        )
+
+TEXTFLAGS_XHTML = (0
+        | TEXT_PRESERVE_LIGATURES
+        | TEXT_PRESERVE_WHITESPACE
+        | TEXT_MEDIABOX_CLIP
+        | TEXT_PRESERVE_IMAGES
+        | TEXT_USE_CID_FOR_UNKNOWN_UNICODE
+        )
+
+TEXTFLAGS_XML = (0
+        | TEXT_PRESERVE_LIGATURES
+        | TEXT_PRESERVE_WHITESPACE
+        | TEXT_MEDIABOX_CLIP
+        | TEXT_USE_CID_FOR_UNKNOWN_UNICODE
+        )
+
+TEXTFLAGS_TEXT = (0
+        | TEXT_PRESERVE_LIGATURES
+        | TEXT_PRESERVE_WHITESPACE
+        | TEXT_MEDIABOX_CLIP
+        | TEXT_USE_CID_FOR_UNKNOWN_UNICODE
+        )
+
+# Simple text encoding options
+TEXT_ENCODING_LATIN = 0
+TEXT_ENCODING_GREEK = 1
+TEXT_ENCODING_CYRILLIC = 2
+
+TOOLS_JM_UNIQUE_ID = 0
+
+# colorspace identifiers
+CS_RGB = 1
+CS_GRAY = 2
+CS_CMYK = 3
+
+# PDF Blend Modes
+PDF_BM_Color = "Color"
+PDF_BM_ColorBurn = "ColorBurn"
+PDF_BM_ColorDodge = "ColorDodge"
+PDF_BM_Darken = "Darken"
+PDF_BM_Difference = "Difference"
+PDF_BM_Exclusion = "Exclusion"
+PDF_BM_HardLight = "HardLight"
+PDF_BM_Hue = "Hue"
+PDF_BM_Lighten = "Lighten"
+PDF_BM_Luminosity = "Luminosity"
+PDF_BM_Multiply = "Multiply"
+PDF_BM_Normal = "Normal"
+PDF_BM_Overlay = "Overlay"
+PDF_BM_Saturation = "Saturation"
+PDF_BM_Screen = "Screen"
+PDF_BM_SoftLight = "Softlight"
+
+
+annot_skel = {
+        "goto1": lambda a, b, c, d, e: f"<>/Rect[{e}]/BS<>/Subtype/Link>>",
+        "goto2": lambda a, b: f"<>/Rect[{b}]/BS<>/Subtype/Link>>",
+        "gotor1": lambda a, b, c, d, e, f, g: f"<>>>/Rect[{g}]/BS<>/Subtype/Link>>",
+        "gotor2": lambda a, b, c: f"<>/Rect[{c}]/BS<>/Subtype/Link>>",
+        "launch": lambda a, b, c: f"<>>>/Rect[{c}]/BS<>/Subtype/Link>>",
+        "uri": lambda a, b: f"<>/Rect[{b}]/BS<>/Subtype/Link>>",
+        "named": lambda a, b: f"<>/Rect[{b}]/BS<>/Subtype/Link>>",
+        }
+
+class FileDataError(RuntimeError):
+    """Raised for documents with file structure issues."""
+    pass
+
+class FileNotFoundError(RuntimeError):
+    """Raised if file does not exist."""
+    pass
+
+class EmptyFileError(FileDataError):
+    """Raised when creating documents from zero-length data."""
+    pass
+
+# propagate exception class to C-level code
+#_set_FileDataError(FileDataError)
+ 
+csRGB = Colorspace(CS_RGB)
+csGRAY = Colorspace(CS_GRAY)
+csCMYK = Colorspace(CS_CMYK)
+
+# These don't appear to be visible in classic, but are used
+# internally.
+#
+dictkey_align = "align"
+dictkey_asc = "ascender"
+dictkey_bidi = "bidi"
+dictkey_bbox = "bbox"
+dictkey_blocks = "blocks"
+dictkey_bpc = "bpc"
+dictkey_c = "c"
+dictkey_chars = "chars"
+dictkey_color = "color"
+dictkey_colorspace = "colorspace"
+dictkey_content = "content"
+dictkey_creationDate = "creationDate"
+dictkey_cs_name = "cs-name"
+dictkey_da = "da"
+dictkey_dashes = "dashes"
+dictkey_descr = "description"
+dictkey_desc = "descender"
+dictkey_dir = "dir"
+dictkey_effect = "effect"
+dictkey_ext = "ext"
+dictkey_filename = "filename"
+dictkey_fill = "fill"
+dictkey_flags = "flags"
+dictkey_char_flags = "char_flags"
+dictkey_font = "font"
+dictkey_glyph = "glyph"
+dictkey_height = "height"
+dictkey_id = "id"
+dictkey_image = "image"
+dictkey_items = "items"
+dictkey_length = "length"
+dictkey_lines = "lines"
+dictkey_matrix = "transform"
+dictkey_modDate = "modDate"
+dictkey_name = "name"
+dictkey_number = "number"
+dictkey_origin = "origin"
+dictkey_rect = "rect"
+dictkey_size = "size"
+dictkey_smask = "smask"
+dictkey_spans = "spans"
+dictkey_stroke = "stroke"
+dictkey_style = "style"
+dictkey_subject = "subject"
+dictkey_text = "text"
+dictkey_title = "title"
+dictkey_type = "type"
+dictkey_ufilename = "ufilename"
+dictkey_width = "width"
+dictkey_wmode = "wmode"
+dictkey_xref = "xref"
+dictkey_xres = "xres"
+dictkey_yres = "yres"
+
+
+try:
+    from pymupdf_fonts import fontdescriptors, fontbuffers
+
+    fitz_fontdescriptors = fontdescriptors.copy()
+    for k in fitz_fontdescriptors.keys():
+        fitz_fontdescriptors[k]["loader"] = fontbuffers[k]
+    del fontdescriptors, fontbuffers
+except ImportError:
+    fitz_fontdescriptors = {}
+
+symbol_glyphs = (   # Glyph list for the built-in font 'Symbol'
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (32, 0.25),
+        (33, 0.333),
+        (34, 0.713),
+        (35, 0.5),
+        (36, 0.549),
+        (37, 0.833),
+        (38, 0.778),
+        (39, 0.439),
+        (40, 0.333),
+        (41, 0.333),
+        (42, 0.5),
+        (43, 0.549),
+        (44, 0.25),
+        (45, 0.549),
+        (46, 0.25),
+        (47, 0.278),
+        (48, 0.5),
+        (49, 0.5),
+        (50, 0.5),
+        (51, 0.5),
+        (52, 0.5),
+        (53, 0.5),
+        (54, 0.5),
+        (55, 0.5),
+        (56, 0.5),
+        (57, 0.5),
+        (58, 0.278),
+        (59, 0.278),
+        (60, 0.549),
+        (61, 0.549),
+        (62, 0.549),
+        (63, 0.444),
+        (64, 0.549),
+        (65, 0.722),
+        (66, 0.667),
+        (67, 0.722),
+        (68, 0.612),
+        (69, 0.611),
+        (70, 0.763),
+        (71, 0.603),
+        (72, 0.722),
+        (73, 0.333),
+        (74, 0.631),
+        (75, 0.722),
+        (76, 0.686),
+        (77, 0.889),
+        (78, 0.722),
+        (79, 0.722),
+        (80, 0.768),
+        (81, 0.741),
+        (82, 0.556),
+        (83, 0.592),
+        (84, 0.611),
+        (85, 0.69),
+        (86, 0.439),
+        (87, 0.768),
+        (88, 0.645),
+        (89, 0.795),
+        (90, 0.611),
+        (91, 0.333),
+        (92, 0.863),
+        (93, 0.333),
+        (94, 0.658),
+        (95, 0.5),
+        (96, 0.5),
+        (97, 0.631),
+        (98, 0.549),
+        (99, 0.549),
+        (100, 0.494),
+        (101, 0.439),
+        (102, 0.521),
+        (103, 0.411),
+        (104, 0.603),
+        (105, 0.329),
+        (106, 0.603),
+        (107, 0.549),
+        (108, 0.549),
+        (109, 0.576),
+        (110, 0.521),
+        (111, 0.549),
+        (112, 0.549),
+        (113, 0.521),
+        (114, 0.549),
+        (115, 0.603),
+        (116, 0.439),
+        (117, 0.576),
+        (118, 0.713),
+        (119, 0.686),
+        (120, 0.493),
+        (121, 0.686),
+        (122, 0.494),
+        (123, 0.48),
+        (124, 0.2),
+        (125, 0.48),
+        (126, 0.549),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (183, 0.46),
+        (160, 0.25),
+        (161, 0.62),
+        (162, 0.247),
+        (163, 0.549),
+        (164, 0.167),
+        (165, 0.713),
+        (166, 0.5),
+        (167, 0.753),
+        (168, 0.753),
+        (169, 0.753),
+        (170, 0.753),
+        (171, 1.042),
+        (172, 0.713),
+        (173, 0.603),
+        (174, 0.987),
+        (175, 0.603),
+        (176, 0.4),
+        (177, 0.549),
+        (178, 0.411),
+        (179, 0.549),
+        (180, 0.549),
+        (181, 0.576),
+        (182, 0.494),
+        (183, 0.46),
+        (184, 0.549),
+        (185, 0.549),
+        (186, 0.549),
+        (187, 0.549),
+        (188, 1),
+        (189, 0.603),
+        (190, 1),
+        (191, 0.658),
+        (192, 0.823),
+        (193, 0.686),
+        (194, 0.795),
+        (195, 0.987),
+        (196, 0.768),
+        (197, 0.768),
+        (198, 0.823),
+        (199, 0.768),
+        (200, 0.768),
+        (201, 0.713),
+        (202, 0.713),
+        (203, 0.713),
+        (204, 0.713),
+        (205, 0.713),
+        (206, 0.713),
+        (207, 0.713),
+        (208, 0.768),
+        (209, 0.713),
+        (210, 0.79),
+        (211, 0.79),
+        (212, 0.89),
+        (213, 0.823),
+        (214, 0.549),
+        (215, 0.549),
+        (216, 0.713),
+        (217, 0.603),
+        (218, 0.603),
+        (219, 1.042),
+        (220, 0.987),
+        (221, 0.603),
+        (222, 0.987),
+        (223, 0.603),
+        (224, 0.494),
+        (225, 0.329),
+        (226, 0.79),
+        (227, 0.79),
+        (228, 0.786),
+        (229, 0.713),
+        (230, 0.384),
+        (231, 0.384),
+        (232, 0.384),
+        (233, 0.384),
+        (234, 0.384),
+        (235, 0.384),
+        (236, 0.494),
+        (237, 0.494),
+        (238, 0.494),
+        (239, 0.494),
+        (183, 0.46),
+        (241, 0.329),
+        (242, 0.274),
+        (243, 0.686),
+        (244, 0.686),
+        (245, 0.686),
+        (246, 0.384),
+        (247, 0.549),
+        (248, 0.384),
+        (249, 0.384),
+        (250, 0.384),
+        (251, 0.384),
+        (252, 0.494),
+        (253, 0.494),
+        (254, 0.494),
+        (183, 0.46),
+        )
+
+
+zapf_glyphs = ( # Glyph list for the built-in font 'ZapfDingbats'
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (32, 0.278),
+        (33, 0.974),
+        (34, 0.961),
+        (35, 0.974),
+        (36, 0.98),
+        (37, 0.719),
+        (38, 0.789),
+        (39, 0.79),
+        (40, 0.791),
+        (41, 0.69),
+        (42, 0.96),
+        (43, 0.939),
+        (44, 0.549),
+        (45, 0.855),
+        (46, 0.911),
+        (47, 0.933),
+        (48, 0.911),
+        (49, 0.945),
+        (50, 0.974),
+        (51, 0.755),
+        (52, 0.846),
+        (53, 0.762),
+        (54, 0.761),
+        (55, 0.571),
+        (56, 0.677),
+        (57, 0.763),
+        (58, 0.76),
+        (59, 0.759),
+        (60, 0.754),
+        (61, 0.494),
+        (62, 0.552),
+        (63, 0.537),
+        (64, 0.577),
+        (65, 0.692),
+        (66, 0.786),
+        (67, 0.788),
+        (68, 0.788),
+        (69, 0.79),
+        (70, 0.793),
+        (71, 0.794),
+        (72, 0.816),
+        (73, 0.823),
+        (74, 0.789),
+        (75, 0.841),
+        (76, 0.823),
+        (77, 0.833),
+        (78, 0.816),
+        (79, 0.831),
+        (80, 0.923),
+        (81, 0.744),
+        (82, 0.723),
+        (83, 0.749),
+        (84, 0.79),
+        (85, 0.792),
+        (86, 0.695),
+        (87, 0.776),
+        (88, 0.768),
+        (89, 0.792),
+        (90, 0.759),
+        (91, 0.707),
+        (92, 0.708),
+        (93, 0.682),
+        (94, 0.701),
+        (95, 0.826),
+        (96, 0.815),
+        (97, 0.789),
+        (98, 0.789),
+        (99, 0.707),
+        (100, 0.687),
+        (101, 0.696),
+        (102, 0.689),
+        (103, 0.786),
+        (104, 0.787),
+        (105, 0.713),
+        (106, 0.791),
+        (107, 0.785),
+        (108, 0.791),
+        (109, 0.873),
+        (110, 0.761),
+        (111, 0.762),
+        (112, 0.762),
+        (113, 0.759),
+        (114, 0.759),
+        (115, 0.892),
+        (116, 0.892),
+        (117, 0.788),
+        (118, 0.784),
+        (119, 0.438),
+        (120, 0.138),
+        (121, 0.277),
+        (122, 0.415),
+        (123, 0.392),
+        (124, 0.392),
+        (125, 0.668),
+        (126, 0.668),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (183, 0.788),
+        (161, 0.732),
+        (162, 0.544),
+        (163, 0.544),
+        (164, 0.91),
+        (165, 0.667),
+        (166, 0.76),
+        (167, 0.76),
+        (168, 0.776),
+        (169, 0.595),
+        (170, 0.694),
+        (171, 0.626),
+        (172, 0.788),
+        (173, 0.788),
+        (174, 0.788),
+        (175, 0.788),
+        (176, 0.788),
+        (177, 0.788),
+        (178, 0.788),
+        (179, 0.788),
+        (180, 0.788),
+        (181, 0.788),
+        (182, 0.788),
+        (183, 0.788),
+        (184, 0.788),
+        (185, 0.788),
+        (186, 0.788),
+        (187, 0.788),
+        (188, 0.788),
+        (189, 0.788),
+        (190, 0.788),
+        (191, 0.788),
+        (192, 0.788),
+        (193, 0.788),
+        (194, 0.788),
+        (195, 0.788),
+        (196, 0.788),
+        (197, 0.788),
+        (198, 0.788),
+        (199, 0.788),
+        (200, 0.788),
+        (201, 0.788),
+        (202, 0.788),
+        (203, 0.788),
+        (204, 0.788),
+        (205, 0.788),
+        (206, 0.788),
+        (207, 0.788),
+        (208, 0.788),
+        (209, 0.788),
+        (210, 0.788),
+        (211, 0.788),
+        (212, 0.894),
+        (213, 0.838),
+        (214, 1.016),
+        (215, 0.458),
+        (216, 0.748),
+        (217, 0.924),
+        (218, 0.748),
+        (219, 0.918),
+        (220, 0.927),
+        (221, 0.928),
+        (222, 0.928),
+        (223, 0.834),
+        (224, 0.873),
+        (225, 0.828),
+        (226, 0.924),
+        (227, 0.924),
+        (228, 0.917),
+        (229, 0.93),
+        (230, 0.931),
+        (231, 0.463),
+        (232, 0.883),
+        (233, 0.836),
+        (234, 0.836),
+        (235, 0.867),
+        (236, 0.867),
+        (237, 0.696),
+        (238, 0.696),
+        (239, 0.874),
+        (183, 0.788),
+        (241, 0.874),
+        (242, 0.76),
+        (243, 0.946),
+        (244, 0.771),
+        (245, 0.865),
+        (246, 0.771),
+        (247, 0.888),
+        (248, 0.967),
+        (249, 0.888),
+        (250, 0.831),
+        (251, 0.873),
+        (252, 0.927),
+        (253, 0.97),
+        (183, 0.788),
+        (183, 0.788),
+        )
+
+
+# Functions
+#
+
+def _read_samples( pixmap, offset, n):
+    # fixme: need to be able to get a sample in one call, as a Python
+    # bytes or similar.
+    ret = []
+    if not pixmap.samples():
+        # mupdf.fz_samples_get() gives a segv if pixmap->samples is null.
+        return ret
+    for i in range( n):
+        ret.append( mupdf.fz_samples_get( pixmap, offset + i))
+    return bytes( ret)
+
+
+def _INRANGE(v, low, high):
+    return low <= v and v <= high
+
+
+def _remove_dest_range(pdf, numbers):
+    pagecount = mupdf.pdf_count_pages(pdf)
+    for i in range(pagecount):
+        n1 = i
+        if n1 in numbers:
+            continue
+
+        pageref = mupdf.pdf_lookup_page_obj( pdf, i)
+        annots = mupdf.pdf_dict_get( pageref, PDF_NAME('Annots'))
+        if not annots.m_internal:
+            continue
+        len_ = mupdf.pdf_array_len(annots)
+        for j in range(len_ - 1, -1, -1):
+            o = mupdf.pdf_array_get( annots, j)
+            if not mupdf.pdf_name_eq( mupdf.pdf_dict_get( o, PDF_NAME('Subtype')), PDF_NAME('Link')):
+                continue
+            action = mupdf.pdf_dict_get( o, PDF_NAME('A'))
+            dest = mupdf.pdf_dict_get( o, PDF_NAME('Dest'))
+            if action.m_internal:
+                if not mupdf.pdf_name_eq( mupdf.pdf_dict_get( action, PDF_NAME('S')), PDF_NAME('GoTo')):
+                    continue
+                dest = mupdf.pdf_dict_get( action, PDF_NAME('D'))
+            pno = -1
+            if mupdf.pdf_is_array( dest):
+                target = mupdf.pdf_array_get( dest, 0)
+                pno = mupdf.pdf_lookup_page_number( pdf, target)
+            elif mupdf.pdf_is_string( dest):
+                location, _, _ = mupdf.fz_resolve_link( pdf.super(), mupdf.pdf_to_text_string( dest))
+                pno = location.page
+            if pno < 0: # page number lookup did not work
+                continue
+            n1 = pno
+            if n1 in numbers:
+                mupdf.pdf_array_delete( annots, j)
+
+
+def ASSERT_PDF(cond):
+    assert isinstance(cond, (mupdf.PdfPage, mupdf.PdfDocument)), f'{type(cond)=} {cond=}'
+    if not cond.m_internal:
+        raise Exception(MSG_IS_NO_PDF)
+
+
+def EMPTY_IRECT():
+    return IRect(FZ_MAX_INF_RECT, FZ_MAX_INF_RECT, FZ_MIN_INF_RECT, FZ_MIN_INF_RECT)
+
+
+def EMPTY_QUAD():
+    return EMPTY_RECT().quad
+
+
+def EMPTY_RECT():
+    return Rect(FZ_MAX_INF_RECT, FZ_MAX_INF_RECT, FZ_MIN_INF_RECT, FZ_MIN_INF_RECT)
+
+
+def ENSURE_OPERATION(pdf):
+    if not JM_have_operation(pdf):
+        raise Exception("No journalling operation started")
+
+
+def INFINITE_IRECT():
+    return IRect(FZ_MIN_INF_RECT, FZ_MIN_INF_RECT, FZ_MAX_INF_RECT, FZ_MAX_INF_RECT)
+
+
+def INFINITE_QUAD():
+    return INFINITE_RECT().quad
+
+
+def INFINITE_RECT():
+    return Rect(FZ_MIN_INF_RECT, FZ_MIN_INF_RECT, FZ_MAX_INF_RECT, FZ_MAX_INF_RECT)
+
+
+def JM_BinFromBuffer(buffer_):
+    '''
+    Turn fz_buffer into a Python bytes object
+    '''
+    assert isinstance(buffer_, mupdf.FzBuffer)
+    ret = mupdf.fz_buffer_extract_copy(buffer_)
+    return ret
+
+
+def JM_EscapeStrFromStr(c):
+    # `c` is typically from SWIG which will have converted a `const char*` from
+    # C into a Python `str` using `PyUnicode_DecodeUTF8(carray, static_cast<
+    # Py_ssize_t >(size), "surrogateescape")`.  This gives us a Python `str`
+    # with some characters encoded as a \0xdcXY sequence, where `XY` are hex
+    # digits for an invalid byte in the original `const char*`.
+    #
+    # This is actually a reasonable way of representing arbitrary
+    # strings from C, but we want to mimic what PyMuPDF does. It uses
+    # `PyUnicode_DecodeRawUnicodeEscape(c, (Py_ssize_t) strlen(c), "replace")`
+    # which gives a string containing actual unicode characters for any invalid
+    # bytes.
+    #
+    # We mimic this by converting the `str` to a `bytes` with 'surrogateescape'
+    # to recognise \0xdcXY sequences, then convert the individual bytes into a
+    # `str` using `chr()`.
+    #
+    # Would be good to have a more efficient way to do this.
+    #
+    if c is None:
+        return ''
+    assert isinstance(c, str), f'{type(c)=}'
+    b = c.encode('utf8', 'surrogateescape')
+    ret = ''
+    for bb in b:
+        ret += chr(bb)
+    return ret
+
+
+def JM_BufferFromBytes(stream):
+    '''
+    Make fz_buffer from a PyBytes, PyByteArray or io.BytesIO object. If a text
+    io.BytesIO, we convert to binary by encoding as utf8.
+    '''
+    if isinstance(stream, (bytes, bytearray)):
+        data = stream
+    elif hasattr(stream, 'getvalue'):
+        data = stream.getvalue()
+        if isinstance(data, str):
+            data = data.encode('utf-8')
+        if not isinstance(data, (bytes, bytearray)):
+            raise Exception(f'.getvalue() returned unexpected type: {type(data)}')
+    else:
+        return mupdf.FzBuffer()
+    return mupdf.fz_new_buffer_from_copied_data(data)
+
+
+def JM_FLOAT_ITEM(obj, idx):
+    if not PySequence_Check(obj):
+        return None
+    return float(obj[idx])
+
+def JM_INT_ITEM(obj, idx):
+    if idx < len(obj):
+        temp = obj[idx]
+        if isinstance(temp, (int, float)):
+            return 0, temp
+    return 1, None
+
+
+def JM_pixmap_from_page(doc, page, ctm, cs, alpha, annots, clip):
+    '''
+    Pixmap creation directly using a short-lived displaylist, so we can support
+    separations.
+    '''
+    SPOTS_NONE = 0
+    SPOTS_OVERPRINT_SIM = 1
+    SPOTS_FULL = 2
+    
+    FZ_ENABLE_SPOT_RENDERING = True # fixme: this is a build-time setting in MuPDF's config.h.
+    if FZ_ENABLE_SPOT_RENDERING:
+        spots = SPOTS_OVERPRINT_SIM
+    else:
+        spots = SPOTS_NONE
+
+    seps = None
+    colorspace = cs
+    
+    matrix = JM_matrix_from_py(ctm)
+    rect = mupdf.fz_bound_page(page)
+    rclip = JM_rect_from_py(clip)
+    rect = mupdf.fz_intersect_rect(rect, rclip) # no-op if clip is not given
+    rect = mupdf.fz_transform_rect(rect, matrix)
+    bbox = mupdf.fz_round_rect(rect)
+
+    # Pixmap of the document's /OutputIntents ("output intents")
+    oi = mupdf.fz_document_output_intent(doc)
+    # if present and compatible, use it instead of the parameter
+    if oi.m_internal:
+        if mupdf.fz_colorspace_n(oi) == mupdf.fz_colorspace_n(cs):
+            colorspace = mupdf.fz_keep_colorspace(oi)
+
+    # check if spots rendering is available and if so use separations
+    if spots != SPOTS_NONE:
+        seps = mupdf.fz_page_separations(page)
+        if seps.m_internal:
+            n = mupdf.fz_count_separations(seps)
+            if spots == SPOTS_FULL:
+                for i in range(n):
+                    mupdf.fz_set_separation_behavior(seps, i, mupdf.FZ_SEPARATION_SPOT)
+            else:
+                for i in range(n):
+                    mupdf.fz_set_separation_behavior(seps, i, mupdf.FZ_SEPARATION_COMPOSITE)
+        elif mupdf.fz_page_uses_overprint(page):
+            # This page uses overprint, so we need an empty
+            # sep object to force the overprint simulation on.
+            seps = mupdf.fz_new_separations(0)
+        elif oi.m_internal and mupdf.fz_colorspace_n(oi) != mupdf.fz_colorspace_n(colorspace):
+            # We have an output intent, and it's incompatible
+            # with the colorspace our device needs. Force the
+            # overprint simulation on, because this ensures that
+            # we 'simulate' the output intent too.
+            seps = mupdf.fz_new_separations(0)
+
+    pix = mupdf.fz_new_pixmap_with_bbox(colorspace, bbox, seps, alpha)
+
+    if alpha:
+        mupdf.fz_clear_pixmap(pix)
+    else:
+        mupdf.fz_clear_pixmap_with_value(pix, 0xFF)
+
+    dev = mupdf.fz_new_draw_device(matrix, pix)
+    if annots:
+        mupdf.fz_run_page(page, dev, mupdf.FzMatrix(), mupdf.FzCookie())
+    else:
+        mupdf.fz_run_page_contents(page, dev, mupdf.FzMatrix(), mupdf.FzCookie())
+    mupdf.fz_close_device(dev)
+    return pix
+
+
+def JM_StrAsChar(x):
+    # fixme: should encode, but swig doesn't pass bytes to C as const char*.
+    return x
+    #return x.encode('utf8')
+
+
+def JM_TUPLE(o: typing.Sequence) -> tuple:
+    return tuple(map(lambda x: round(x, 5) if abs(x) >= 1e-4 else 0, o))
+
+
+def JM_TUPLE3(o: typing.Sequence) -> tuple:
+    return tuple(map(lambda x: round(x, 3) if abs(x) >= 1e-3 else 0, o))
+
+
+def JM_UnicodeFromStr(s):
+    if s is None:
+        return ''
+    if isinstance(s, bytes):
+        s = s.decode('utf8')
+    assert isinstance(s, str), f'{type(s)=} {s=}'
+    return s
+
+
+def JM_add_annot_id(annot, stem):
+    '''
+    Add a unique /NM key to an annotation or widget.
+    Append a number to 'stem' such that the result is a unique name.
+    '''
+    assert isinstance(annot, mupdf.PdfAnnot)
+    page = _pdf_annot_page(annot)
+    annot_obj = mupdf.pdf_annot_obj( annot)
+    names = JM_get_annot_id_list(page)
+    i = 0
+    while 1:
+        stem_id = f'{JM_annot_id_stem}-{stem}{i}'
+        if stem_id not in names:
+            break
+        i += 1
+    response = JM_StrAsChar(stem_id)
+    name = mupdf.pdf_new_string( response, len(response))
+    mupdf.pdf_dict_puts(annot_obj, "NM", name)
+    page.doc().m_internal.resynth_required = 0
+
+
+def JM_add_oc_object(pdf, ref, xref):
+    '''
+    Add OC object reference to a dictionary
+    '''
+    indobj = mupdf.pdf_new_indirect(pdf, xref, 0)
+    if not mupdf.pdf_is_dict(indobj):
+        RAISEPY(MSG_BAD_OC_REF, PyExc_ValueError)
+    type_ = mupdf.pdf_dict_get(indobj, PDF_NAME('Type'))
+    if (mupdf.pdf_objcmp(type_, PDF_NAME('OCG')) == 0
+            or mupdf.pdf_objcmp(type_, PDF_NAME('OCMD')) == 0
+            ):
+        mupdf.pdf_dict_put(ref, PDF_NAME('OC'), indobj)
+    else:
+        RAISEPY(MSG_BAD_OC_REF, PyExc_ValueError)
+
+
+def JM_annot_border(annot_obj):
+    dash_py = list()
+    style = None
+    width = -1
+    clouds = -1
+    obj = None
+
+    obj = mupdf.pdf_dict_get( annot_obj, PDF_NAME('Border'))
+    if mupdf.pdf_is_array( obj):
+        width = mupdf.pdf_to_real( mupdf.pdf_array_get( obj, 2))
+        if mupdf.pdf_array_len( obj) == 4:
+            dash = mupdf.pdf_array_get( obj, 3)
+            for i in range( mupdf.pdf_array_len( dash)):
+                val = mupdf.pdf_to_int( mupdf.pdf_array_get( dash, i))
+                dash_py.append( val)
+
+    bs_o = mupdf.pdf_dict_get( annot_obj, PDF_NAME('BS'))
+    if bs_o.m_internal:
+        width = mupdf.pdf_to_real( mupdf.pdf_dict_get( bs_o, PDF_NAME('W')))
+        style = mupdf.pdf_to_name( mupdf.pdf_dict_get( bs_o, PDF_NAME('S')))
+        if style == '':
+            style = None
+        obj = mupdf.pdf_dict_get( bs_o, PDF_NAME('D'))
+        if obj.m_internal:
+            for i in range( mupdf.pdf_array_len( obj)):
+                val = mupdf.pdf_to_int( mupdf.pdf_array_get( obj, i))
+                dash_py.append( val)
+
+    obj = mupdf.pdf_dict_get( annot_obj, PDF_NAME('BE'))
+    if obj.m_internal:
+        clouds = mupdf.pdf_to_int( mupdf.pdf_dict_get( obj, PDF_NAME('I')))
+
+    res = dict()
+    res[ dictkey_width] = width
+    res[ dictkey_dashes] = tuple( dash_py)
+    res[ dictkey_style] = style
+    res[ 'clouds'] = clouds
+    return res
+
+
+def JM_annot_colors(annot_obj):
+    res = dict()
+    bc = list() # stroke colors
+    fc =list()  # fill colors
+    o = mupdf.pdf_dict_get(annot_obj, mupdf.PDF_ENUM_NAME_C)
+    if mupdf.pdf_is_array(o):
+        n = mupdf.pdf_array_len(o)
+        for i in range(n):
+            col = mupdf.pdf_to_real( mupdf.pdf_array_get(o, i))
+            bc.append(col)
+    res[dictkey_stroke] = bc
+
+    o = mupdf.pdf_dict_gets(annot_obj, "IC")
+    if mupdf.pdf_is_array(o):
+        n = mupdf.pdf_array_len(o)
+        for i in range(n):
+            col = mupdf.pdf_to_real( mupdf.pdf_array_get(o, i))
+            fc.append(col)
+
+    res[dictkey_fill] = fc
+    return res
+
+
+def JM_annot_set_border( border, doc, annot_obj):
+    assert isinstance(border, dict)
+    obj = None
+    dashlen = 0
+    nwidth = border.get( dictkey_width)     # new width
+    ndashes = border.get( dictkey_dashes)   # new dashes
+    nstyle = border.get( dictkey_style)     # new style
+    nclouds  = border.get( 'clouds', -1)    # new clouds value
+
+    # get old border properties
+    oborder = JM_annot_border( annot_obj)
+
+    # delete border-related entries
+    mupdf.pdf_dict_del( annot_obj, PDF_NAME('BS'))
+    mupdf.pdf_dict_del( annot_obj, PDF_NAME('BE'))
+    mupdf.pdf_dict_del( annot_obj, PDF_NAME('Border'))
+
+    # populate border items: keep old values for any omitted new ones
+    if nwidth < 0:
+        nwidth = oborder.get( dictkey_width)    # no new width: keep current
+    if ndashes is None:
+        ndashes = oborder.get( dictkey_dashes)  # no new dashes: keep old
+    if nstyle is None:
+        nstyle  = oborder.get( dictkey_style)   # no new style: keep old
+    if nclouds < 0:
+        nclouds  = oborder.get( "clouds", -1)   # no new clouds: keep old
+
+    if isinstance( ndashes, tuple) and len( ndashes) > 0:
+        dashlen = len( ndashes)
+        darr = mupdf.pdf_new_array( doc, dashlen)
+        for d in ndashes:
+            mupdf.pdf_array_push_int( darr, d)
+        mupdf.pdf_dict_putl( annot_obj, darr, PDF_NAME('BS'), PDF_NAME('D'))
+
+    mupdf.pdf_dict_putl(
+            annot_obj,
+            mupdf.pdf_new_real( nwidth),
+            PDF_NAME('BS'),
+            PDF_NAME('W'),
+            )
+
+    if dashlen == 0:
+        obj = JM_get_border_style( nstyle)
+    else:
+        obj = PDF_NAME('D')
+    mupdf.pdf_dict_putl( annot_obj, obj, PDF_NAME('BS'), PDF_NAME('S'))
+
+    if nclouds > 0:
+        mupdf.pdf_dict_put_dict( annot_obj, PDF_NAME('BE'), 2)
+        obj = mupdf.pdf_dict_get( annot_obj, PDF_NAME('BE'))
+        mupdf.pdf_dict_put( obj, PDF_NAME('S'), PDF_NAME('C'))
+        mupdf.pdf_dict_put_int( obj, PDF_NAME('I'), nclouds)
+
+
+def make_escape(ch):
+    if ch == 92:
+        return "\\u005c"
+    elif 32 <= ch <= 127 or ch == 10:
+        return chr(ch)
+    elif 0xd800 <= ch <= 0xdfff:  # orphaned surrogate
+        return "\\ufffd"
+    elif ch <= 0xffff:
+        return "\\u%04x" % ch
+    else:
+        return "\\U%08x" % ch
+
+
+def JM_append_rune(buff, ch):
+    """
+    APPEND non-ascii runes in unicode escape format to fz_buffer.
+    """
+    mupdf.fz_append_string(buff, make_escape(ch))
+
+
+def JM_append_word(lines, buff, wbbox, block_n, line_n, word_n):
+    '''
+    Functions for wordlist output
+    '''
+    s = JM_EscapeStrFromBuffer(buff)
+    litem = (
+            wbbox.x0,
+            wbbox.y0,
+            wbbox.x1,
+            wbbox.y1,
+            s,
+            block_n,
+            line_n,
+            word_n,
+            )
+    lines.append(litem)
+    return word_n + 1, mupdf.FzRect(mupdf.FzRect.Fixed_EMPTY)   # word counter
+
+
+def JM_add_layer_config( pdf, name, creator, ON):
+    '''
+    Add OC configuration to the PDF catalog
+    '''
+    ocp = JM_ensure_ocproperties( pdf)
+    configs = mupdf.pdf_dict_get( ocp, PDF_NAME('Configs'))
+    if not mupdf.pdf_is_array( configs):
+        configs = mupdf.pdf_dict_put_array( ocp, PDF_NAME('Configs'), 1)
+    D = mupdf.pdf_new_dict( pdf, 5)
+    mupdf.pdf_dict_put_text_string( D, PDF_NAME('Name'), name)
+    if creator is not None:
+        mupdf.pdf_dict_put_text_string( D, PDF_NAME('Creator'), creator)
+    mupdf.pdf_dict_put( D, PDF_NAME('BaseState'), PDF_NAME('OFF'))
+    onarray = mupdf.pdf_dict_put_array( D, PDF_NAME('ON'), 5)
+    if not ON:
+        pass
+    else:
+        ocgs = mupdf.pdf_dict_get( ocp, PDF_NAME('OCGs'))
+        n = len(ON)
+        for i in range(n):
+            xref = 0
+            e, xref = JM_INT_ITEM(ON, i)
+            if e == 1:
+                continue
+            ind = mupdf.pdf_new_indirect( pdf, xref, 0)
+            if mupdf.pdf_array_contains( ocgs, ind):
+                mupdf.pdf_array_push( onarray, ind)
+    mupdf.pdf_array_push( configs, D)
+
+
+def JM_char_bbox(line, ch):
+    '''
+    return rect of char quad
+    '''
+    q = JM_char_quad(line, ch)
+    r = mupdf.fz_rect_from_quad(q)
+    if not line.m_internal.wmode:
+        return r
+    if r.y1 < r.y0 + ch.m_internal.size:
+        r.y0 = r.y1 - ch.m_internal.size
+    return r
+
+
+def JM_char_font_flags(font, line, ch):
+    flags = 0
+    if line and ch:
+        flags += detect_super_script(line, ch)
+    flags += mupdf.fz_font_is_italic(font) * TEXT_FONT_ITALIC
+    flags += mupdf.fz_font_is_serif(font) * TEXT_FONT_SERIFED
+    flags += mupdf.fz_font_is_monospaced(font) * TEXT_FONT_MONOSPACED
+    flags += mupdf.fz_font_is_bold(font) * TEXT_FONT_BOLD
+    return flags
+
+
+def JM_char_quad(line, ch):
+    '''
+    re-compute char quad if ascender/descender values make no sense
+    '''
+    if 1 and g_use_extra:
+        # This reduces time taken to extract text from PyMuPDF.pdf from 20s to
+        # 15s.
+        return mupdf.FzQuad(extra.JM_char_quad( line.m_internal, ch.m_internal))
+        
+    assert isinstance(line, mupdf.FzStextLine)
+    assert isinstance(ch, mupdf.FzStextChar)
+    if _globals.skip_quad_corrections:   # no special handling
+        return ch.quad
+    if line.m_internal.wmode:  # never touch vertical write mode
+        return ch.quad
+    font = mupdf.FzFont(mupdf.ll_fz_keep_font(ch.m_internal.font))
+    asc = JM_font_ascender(font)
+    dsc = JM_font_descender(font)
+    fsize = ch.m_internal.size
+    asc_dsc = asc - dsc + FLT_EPSILON
+    if asc_dsc >= 1 and _globals.small_glyph_heights == 0:   # no problem
+        return mupdf.FzQuad(ch.m_internal.quad)
+
+    # Re-compute quad with adjusted ascender / descender values:
+    # Move ch->origin to (0,0) and de-rotate quad, then adjust the corners,
+    # re-rotate and move back to ch->origin location.
+    fsize = ch.m_internal.size
+    bbox = mupdf.fz_font_bbox(font)
+    fwidth = bbox.x1 - bbox.x0
+    if asc < 1e-3:  # probably Tesseract glyphless font
+        dsc = -0.1
+        asc = 0.9
+        asc_dsc = 1.0
+    
+    if _globals.small_glyph_heights or asc_dsc < 1:
+        dsc = dsc / asc_dsc
+        asc = asc / asc_dsc
+    asc_dsc = asc - dsc
+    asc = asc * fsize / asc_dsc
+    dsc = dsc * fsize / asc_dsc
+    
+    # Re-compute quad with the adjusted ascender / descender values:
+    # Move ch->origin to (0,0) and de-rotate quad, then adjust the corners,
+    # re-rotate and move back to ch->origin location.
+    c = line.m_internal.dir.x  # cosine
+    s = line.m_internal.dir.y  # sine
+    trm1 = mupdf.fz_make_matrix(c, -s, s, c, 0, 0) # derotate
+    trm2 = mupdf.fz_make_matrix(c, s, -s, c, 0, 0) # rotate
+    if (c == -1):   # left-right flip
+        trm1.d = 1
+        trm2.d = 1
+    xlate1 = mupdf.fz_make_matrix(1, 0, 0, 1, -ch.m_internal.origin.x, -ch.m_internal.origin.y)
+    xlate2 = mupdf.fz_make_matrix(1, 0, 0, 1, ch.m_internal.origin.x, ch.m_internal.origin.y)
+
+    quad = mupdf.fz_transform_quad(mupdf.FzQuad(ch.m_internal.quad), xlate1)    # move origin to (0,0)
+    quad = mupdf.fz_transform_quad(quad, trm1) # de-rotate corners
+    
+    # adjust vertical coordinates
+    if c == 1 and quad.ul.y > 0:    # up-down flip
+        quad.ul.y = asc
+        quad.ur.y = asc
+        quad.ll.y = dsc
+        quad.lr.y = dsc
+    else:
+        quad.ul.y = -asc
+        quad.ur.y = -asc
+        quad.ll.y = -dsc
+        quad.lr.y = -dsc
+
+    # adjust horizontal coordinates that are too crazy:
+    # (1) left x must be >= 0
+    # (2) if bbox width is 0, lookup char advance in font.
+    if quad.ll.x < 0:
+        quad.ll.x = 0
+        quad.ul.x = 0
+    
+    cwidth = quad.lr.x - quad.ll.x
+    if cwidth < FLT_EPSILON:
+        glyph = mupdf.fz_encode_character( font, ch.m_internal.c)
+        if glyph:
+            fwidth = mupdf.fz_advance_glyph( font, glyph, line.m_internal.wmode)
+            quad.lr.x = quad.ll.x + fwidth * fsize
+            quad.ur.x = quad.lr.x
+
+    quad = mupdf.fz_transform_quad(quad, trm2) # rotate back
+    quad = mupdf.fz_transform_quad(quad, xlate2)   # translate back
+    return quad
+
+
+def JM_choice_options(annot):
+    '''
+    return list of choices for list or combo boxes
+    '''
+    annot_obj = mupdf.pdf_annot_obj( annot.this)
+    
+    opts = mupdf.pdf_choice_widget_options2( annot, 0)
+    n = len( opts)
+    if n == 0:
+        return  # wrong widget type
+
+    optarr = mupdf.pdf_dict_get( annot_obj, PDF_NAME('Opt'))
+    liste = []
+
+    for i in range( n):
+        m = mupdf.pdf_array_len( mupdf.pdf_array_get( optarr, i))
+        if m == 2:
+            val = (
+                    mupdf.pdf_to_text_string( mupdf.pdf_array_get( mupdf.pdf_array_get( optarr, i), 0)),
+                    mupdf.pdf_to_text_string( mupdf.pdf_array_get( mupdf.pdf_array_get( optarr, i), 1)),
+                    )
+            liste.append( val)
+        else:
+            val = mupdf.pdf_to_text_string( mupdf.pdf_array_get( optarr, i))
+            liste.append( val)
+    return liste
+
+
+def JM_clear_pixmap_rect_with_value(dest, value, b):
+    '''
+    Clear a pixmap rectangle - my version also supports non-alpha pixmaps
+    '''
+    b = mupdf.fz_intersect_irect(b, mupdf.fz_pixmap_bbox(dest))
+    w = b.x1 - b.x0
+    y = b.y1 - b.y0
+    if w <= 0 or y <= 0:
+        return 0
+
+    destspan = dest.stride()
+    destp = destspan * (b.y0 - dest.y()) + dest.n() * (b.x0 - dest.x())
+
+    # CMYK needs special handling (and potentially any other subtractive colorspaces)
+    if mupdf.fz_colorspace_n(dest.colorspace()) == 4:
+        value = 255 - value
+        while 1:
+            s = destp
+            for x in range(0, w):
+                mupdf.fz_samples_set(dest, s, 0)
+                s += 1
+                mupdf.fz_samples_set(dest, s, 0)
+                s += 1
+                mupdf.fz_samples_set(dest, s, 0)
+                s += 1
+                mupdf.fz_samples_set(dest, s, value)
+                s += 1
+                if dest.alpha():
+                    mupdf.fz_samples_set(dest, s, 255)
+                    s += 1
+            destp += destspan
+            if y == 0:
+                break
+            y -= 1
+        return 1
+
+    while 1:
+        s = destp
+        for x in range(w):
+            for k in range(dest.n()-1):
+                mupdf.fz_samples_set(dest, s, value)
+                s += 1
+            if dest.alpha():
+                mupdf.fz_samples_set(dest, s, 255)
+                s += 1
+            else:
+                mupdf.fz_samples_set(dest, s, value)
+                s += 1
+        destp += destspan
+        if y == 0:
+            break
+        y -= 1
+    return 1
+
+
+def JM_color_FromSequence(color):
+    
+    if isinstance(color, (int, float)):    # maybe just a single float
+        color = [color]
+    
+    if not isinstance( color, (list, tuple)):
+        return -1, []
+    
+    if len(color) not in (0, 1, 3, 4):
+        return -1, []
+    
+    ret = color[:]
+    for i in range(len(ret)):
+        if ret[i] < 0 or ret[i] > 1:
+            ret[i] = 1
+    return len(ret), ret
+
+
+def JM_color_count( pm, clip):
+    if g_use_extra:
+        return extra.ll_JM_color_count(pm.m_internal, clip)
+    
+    rc = dict()
+    cnt = 0
+    irect = mupdf.fz_pixmap_bbox( pm)
+    irect = mupdf.fz_intersect_irect(irect, mupdf.fz_round_rect(JM_rect_from_py(clip)))
+    stride = pm.stride()
+    width = irect.x1 - irect.x0
+    height = irect.y1 - irect.y0
+    n = pm.n()
+    substride = width * n
+    s = stride * (irect.y0 - pm.y()) + (irect.x0 - pm.x()) * n
+    oldpix = _read_samples( pm, s, n)
+    cnt = 0
+    if mupdf.fz_is_empty_irect(irect):
+        return rc
+    for i in range( height):
+        for j in range( 0, substride, n):
+            newpix = _read_samples( pm, s + j, n)
+            if newpix != oldpix:
+                pixel = oldpix
+                c = rc.get( pixel, None)
+                if c is not None:
+                    cnt += c
+                rc[ pixel] = cnt
+                cnt = 1
+                oldpix = newpix
+            else:
+                cnt += 1
+        s += stride
+    pixel = oldpix
+    c = rc.get( pixel)
+    if c is not None:
+        cnt += c
+    rc[ pixel] = cnt
+    return rc
+
+
+def JM_compress_buffer(inbuffer):
+    '''
+    compress char* into a new buffer
+    '''
+    data, compressed_length = mupdf.fz_new_deflated_data_from_buffer(
+            inbuffer,
+            mupdf.FZ_DEFLATE_BEST,
+            )
+    #log( '{=data compressed_length}')
+    if not data or compressed_length == 0:
+        return None
+    buf = mupdf.FzBuffer(mupdf.fz_new_buffer_from_data(data, compressed_length))
+    mupdf.fz_resize_buffer(buf, compressed_length)
+    return buf
+
+
+def JM_copy_rectangle(page, area):
+    need_new_line = 0
+    buffer = io.StringIO()
+    for block in page:
+        if block.m_internal.type != mupdf.FZ_STEXT_BLOCK_TEXT:
+            continue
+        for line in block:
+            line_had_text = 0
+            for ch in line:
+                r = JM_char_bbox(line, ch)
+                if JM_rects_overlap(area, r):
+                    line_had_text = 1
+                    if need_new_line:
+                        buffer.write("\n")
+                        need_new_line = 0
+                    buffer.write(make_escape(ch.m_internal.c))
+            if line_had_text:
+                need_new_line = 1
+
+    s = buffer.getvalue()   # take over the data
+    return s
+
+
+def JM_convert_to_pdf(doc, fp, tp, rotate):
+    '''
+    Convert any MuPDF document to a PDF
+    Returns bytes object containing the PDF, created via 'write' function.
+    '''
+    pdfout = mupdf.PdfDocument()
+    incr = 1
+    s = fp
+    e = tp
+    if fp > tp:
+        incr = -1   # count backwards
+        s = tp      # adjust ...
+        e = fp      # ... range
+    rot = JM_norm_rotation(rotate)
+    i = fp
+    while 1:    # interpret & write document pages as PDF pages
+        if not _INRANGE(i, s, e):
+            break
+        page = mupdf.fz_load_page(doc, i)
+        mediabox = mupdf.fz_bound_page(page)
+        dev, resources, contents = mupdf.pdf_page_write(pdfout, mediabox)
+        mupdf.fz_run_page(page, dev, mupdf.FzMatrix(), mupdf.FzCookie())
+        mupdf.fz_close_device(dev)
+        dev = None
+        page_obj = mupdf.pdf_add_page(pdfout, mediabox, rot, resources, contents)
+        mupdf.pdf_insert_page(pdfout, -1, page_obj)
+        i += incr
+    # PDF created - now write it to Python bytearray
+    # prepare write options structure
+    opts = mupdf.PdfWriteOptions()
+    opts.do_garbage         = 4
+    opts.do_compress        = 1
+    opts.do_compress_images = 1
+    opts.do_compress_fonts  = 1
+    opts.do_sanitize        = 1
+    opts.do_incremental     = 0
+    opts.do_ascii           = 0
+    opts.do_decompress      = 0
+    opts.do_linear          = 0
+    opts.do_clean           = 1
+    opts.do_pretty          = 0
+
+    res = mupdf.fz_new_buffer(8192)
+    out = mupdf.FzOutput(res)
+    mupdf.pdf_write_document(pdfout, out, opts)
+    out.fz_close_output()
+    c = mupdf.fz_buffer_extract_copy(res)
+    assert isinstance(c, bytes)
+    return c
+
+
+# Copied from MuPDF v1.14
+# Create widget
+def JM_create_widget(doc, page, type, fieldname):
+    old_sigflags = mupdf.pdf_to_int(mupdf.pdf_dict_getp(mupdf.pdf_trailer(doc), "Root/AcroForm/SigFlags"))
+    #log( '*** JM_create_widget()')
+    #log( f'{mupdf.pdf_create_annot_raw=}')
+    #log( f'{page=}')
+    #log( f'{mupdf.PDF_ANNOT_WIDGET=}')
+    annot = mupdf.pdf_create_annot_raw(page, mupdf.PDF_ANNOT_WIDGET)
+    annot_obj = mupdf.pdf_annot_obj(annot)
+    try:
+        JM_set_field_type(doc, annot_obj, type)
+        mupdf.pdf_dict_put_text_string(annot_obj, PDF_NAME('T'), fieldname)
+
+        if type == mupdf.PDF_WIDGET_TYPE_SIGNATURE:
+            sigflags = old_sigflags | (SigFlag_SignaturesExist | SigFlag_AppendOnly)
+            mupdf.pdf_dict_putl(
+                    mupdf.pdf_trailer(doc),
+                    mupdf.pdf_new_int(sigflags),
+                    PDF_NAME('Root'),
+                    PDF_NAME('AcroForm'),
+                    PDF_NAME('SigFlags'),
+                    )
+        # pdf_create_annot will have linked the new widget into the page's
+        # annot array. We also need it linked into the document's form
+        form = mupdf.pdf_dict_getp(mupdf.pdf_trailer(doc), "Root/AcroForm/Fields")
+        if not form.m_internal:
+            form = mupdf.pdf_new_array(doc, 1)
+            mupdf.pdf_dict_putl(
+                    mupdf.pdf_trailer(doc),
+                    form,
+                    PDF_NAME('Root'),
+                    PDF_NAME('AcroForm'),
+                    PDF_NAME('Fields'),
+                    )
+        mupdf.pdf_array_push(form, annot_obj)  # Cleanup relies on this statement being last
+    except Exception:
+        if g_exceptions_verbose:    exception_info()
+        mupdf.pdf_delete_annot(page, annot)
+
+        if type == mupdf.PDF_WIDGET_TYPE_SIGNATURE:
+            mupdf.pdf_dict_putl(
+                    mupdf.pdf_trailer(doc),
+                    mupdf.pdf_new_int(old_sigflags),
+                    PDF_NAME('Root'),
+                    PDF_NAME('AcroForm'),
+                    PDF_NAME('SigFlags'),
+                    )
+        raise
+    return annot
+
+
+def JM_cropbox(page_obj):
+    '''
+    return a PDF page's CropBox
+    '''
+    if g_use_extra:
+        return extra.JM_cropbox(page_obj)
+    
+    mediabox = JM_mediabox(page_obj)
+    cropbox = mupdf.pdf_to_rect(
+                mupdf.pdf_dict_get_inheritable(page_obj, PDF_NAME('CropBox'))
+                )
+    if mupdf.fz_is_infinite_rect(cropbox) or mupdf.fz_is_empty_rect(cropbox):
+        cropbox = mediabox
+    y0 = mediabox.y1 - cropbox.y1
+    y1 = mediabox.y1 - cropbox.y0
+    cropbox.y0 = y0
+    cropbox.y1 = y1
+    return cropbox
+
+
+def JM_cropbox_size(page_obj):
+    rect = JM_cropbox(page_obj)
+    w = abs(rect.x1 - rect.x0)
+    h = abs(rect.y1 - rect.y0)
+    size = mupdf.fz_make_point(w, h)
+    return size
+
+
+def JM_derotate_page_matrix(page):
+    '''
+    just the inverse of rotation
+    '''
+    mp = JM_rotate_page_matrix(page)
+    return mupdf.fz_invert_matrix(mp)
+
+
+def JM_embed_file(
+        pdf,
+        buf,
+        filename,
+        ufilename,
+        desc,
+        compress,
+        ):
+    '''
+    embed a new file in a PDF (not only /EmbeddedFiles entries)
+    '''
+    len_ = 0
+    val = mupdf.pdf_new_dict(pdf, 6)
+    mupdf.pdf_dict_put_dict(val, PDF_NAME('CI'), 4)
+    ef = mupdf.pdf_dict_put_dict(val, PDF_NAME('EF'), 4)
+    mupdf.pdf_dict_put_text_string(val, PDF_NAME('F'), filename)
+    mupdf.pdf_dict_put_text_string(val, PDF_NAME('UF'), ufilename)
+    mupdf.pdf_dict_put_text_string(val, PDF_NAME('Desc'), desc)
+    mupdf.pdf_dict_put(val, PDF_NAME('Type'), PDF_NAME('Filespec'))
+    bs = b'  '
+    f = mupdf.pdf_add_stream(
+            pdf,
+            #mupdf.fz_fz_new_buffer_from_copied_data(bs),
+            mupdf.fz_new_buffer_from_copied_data(bs),
+            mupdf.PdfObj(),
+            0,
+            )
+    mupdf.pdf_dict_put(ef, PDF_NAME('F'), f)
+    JM_update_stream(pdf, f, buf, compress)
+    len_, _ = mupdf.fz_buffer_storage(buf)
+    mupdf.pdf_dict_put_int(f, PDF_NAME('DL'), len_)
+    mupdf.pdf_dict_put_int(f, PDF_NAME('Length'), len_)
+    params = mupdf.pdf_dict_put_dict(f, PDF_NAME('Params'), 4)
+    mupdf.pdf_dict_put_int(params, PDF_NAME('Size'), len_)
+    return val
+
+
+def JM_embedded_clean(pdf):
+    '''
+    perform some cleaning if we have /EmbeddedFiles:
+    (1) remove any /Limits if /Names exists
+    (2) remove any empty /Collection
+    (3) set /PageMode/UseAttachments
+    '''
+    root = mupdf.pdf_dict_get( mupdf.pdf_trailer( pdf), PDF_NAME('Root'))
+
+    # remove any empty /Collection entry
+    coll = mupdf.pdf_dict_get(root, PDF_NAME('Collection'))
+    if coll.m_internal and mupdf.pdf_dict_len(coll) == 0:
+        mupdf.pdf_dict_del(root, PDF_NAME('Collection'))
+
+    efiles = mupdf.pdf_dict_getl(
+            root,
+            PDF_NAME('Names'),
+            PDF_NAME('EmbeddedFiles'),
+            PDF_NAME('Names'),
+            )
+    if efiles.m_internal:
+        mupdf.pdf_dict_put_name(root, PDF_NAME('PageMode'), "UseAttachments")
+
+
+def JM_EscapeStrFromBuffer(buff):
+    if not buff.m_internal:
+        return ''
+    s = mupdf.fz_buffer_extract_copy(buff)
+    val = PyUnicode_DecodeRawUnicodeEscape(s, errors='replace')
+    return val
+
+
+def JM_ensure_identity(pdf):
+    '''
+    Store ID in PDF trailer
+    '''
+    id_ = mupdf.pdf_dict_get( mupdf.pdf_trailer(pdf), PDF_NAME('ID'))
+    if not id_.m_internal:
+        rnd0 = mupdf.fz_memrnd2(16)
+        # Need to convert raw bytes into a str to send to
+        # mupdf.pdf_new_string(). chr() seems to work for this.
+        rnd = ''
+        for i in rnd0:
+            rnd += chr(i)
+        id_ = mupdf.pdf_dict_put_array( mupdf.pdf_trailer( pdf), PDF_NAME('ID'), 2)
+        mupdf.pdf_array_push( id_, mupdf.pdf_new_string( rnd, len(rnd)))
+        mupdf.pdf_array_push( id_, mupdf.pdf_new_string( rnd, len(rnd)))
+
+def JM_ensure_ocproperties(pdf):
+    '''
+    Ensure OCProperties, return /OCProperties key
+    '''
+    ocp = mupdf.pdf_dict_get(mupdf.pdf_dict_get(mupdf.pdf_trailer(pdf), PDF_NAME('Root')), PDF_NAME('OCProperties'))
+    if ocp.m_internal:
+        return ocp
+    root = mupdf.pdf_dict_get(mupdf.pdf_trailer(pdf), PDF_NAME('Root'))
+    ocp = mupdf.pdf_dict_put_dict(root, PDF_NAME('OCProperties'), 2)
+    mupdf.pdf_dict_put_array(ocp, PDF_NAME('OCGs'), 0)
+    D = mupdf.pdf_dict_put_dict(ocp, PDF_NAME('D'), 5)
+    mupdf.pdf_dict_put_array(D, PDF_NAME('ON'), 0)
+    mupdf.pdf_dict_put_array(D, PDF_NAME('OFF'), 0)
+    mupdf.pdf_dict_put_array(D, PDF_NAME('Order'), 0)
+    mupdf.pdf_dict_put_array(D, PDF_NAME('RBGroups'), 0)
+    return ocp
+
+
+def JM_expand_fname(name):
+    '''
+    Make /DA string of annotation
+    '''
+    if not name:    return "Helv"
+    if name.startswith("Co"):   return "Cour"
+    if name.startswith("co"):   return "Cour"
+    if name.startswith("Ti"):   return "TiRo"
+    if name.startswith("ti"):   return "TiRo"
+    if name.startswith("Sy"):   return "Symb"
+    if name.startswith("sy"):   return "Symb"
+    if name.startswith("Za"):   return "ZaDb"
+    if name.startswith("za"):   return "ZaDb"
+    return "Helv"
+
+
+def JM_field_type_text(wtype):
+    '''
+    String from widget type
+    '''
+    if wtype == mupdf.PDF_WIDGET_TYPE_BUTTON:
+        return "Button"
+    if wtype == mupdf.PDF_WIDGET_TYPE_CHECKBOX:
+        return "CheckBox"
+    if wtype == mupdf.PDF_WIDGET_TYPE_RADIOBUTTON:
+        return "RadioButton"
+    if wtype == mupdf.PDF_WIDGET_TYPE_TEXT:
+        return "Text"
+    if wtype == mupdf.PDF_WIDGET_TYPE_LISTBOX:
+        return "ListBox"
+    if wtype == mupdf.PDF_WIDGET_TYPE_COMBOBOX:
+        return "ComboBox"
+    if wtype == mupdf.PDF_WIDGET_TYPE_SIGNATURE:
+        return "Signature"
+    return "unknown"
+
+
+def JM_fill_pixmap_rect_with_color(dest, col, b):
+    assert isinstance(dest, mupdf.FzPixmap)
+    # fill a rect with a color tuple
+    b = mupdf.fz_intersect_irect(b, mupdf.fz_pixmap_bbox( dest))
+    w = b.x1 - b.x0
+    y = b.y1 - b.y0
+    if w <= 0 or y <= 0:
+        return 0
+    destspan = dest.stride()
+    destp = destspan * (b.y0 - dest.y()) + dest.n() * (b.x0 - dest.x())
+    while 1:
+        s = destp
+        for x in range(w):
+            for i in range( dest.n()):
+                mupdf.fz_samples_set(dest, s, col[i])
+                s += 1
+        destp += destspan
+        y -= 1
+        if y == 0:
+            break
+    return 1
+
+
+def JM_find_annot_irt(annot):
+    '''
+    Return the first annotation whose /IRT key ("In Response To") points to
+    annot. Used to remove the response chain of a given annotation.
+    '''
+    assert isinstance(annot, mupdf.PdfAnnot)
+    irt_annot = None    # returning this
+    annot_obj = mupdf.pdf_annot_obj(annot)
+    found = 0
+    # loop thru MuPDF's internal annots array
+    page = _pdf_annot_page(annot)
+    irt_annot = mupdf.pdf_first_annot(page)
+    while 1:
+        assert isinstance(irt_annot, mupdf.PdfAnnot)
+        if not irt_annot.m_internal:
+            break
+        irt_annot_obj = mupdf.pdf_annot_obj(irt_annot)
+        o = mupdf.pdf_dict_gets(irt_annot_obj, 'IRT')
+        if o.m_internal:
+            if not mupdf.pdf_objcmp(o, annot_obj):
+                found = 1
+                break
+        irt_annot = mupdf.pdf_next_annot(irt_annot)
+    if found:
+        return irt_annot
+
+
+def JM_font_ascender(font):
+    '''
+    need own versions of ascender / descender
+    '''
+    assert isinstance(font, mupdf.FzFont)
+    if _globals.skip_quad_corrections:
+        return 0.8
+    return mupdf.fz_font_ascender(font)
+
+
+def JM_font_descender(font):
+    '''
+    need own versions of ascender / descender
+    '''
+    assert isinstance(font, mupdf.FzFont)
+    if _globals.skip_quad_corrections:
+        return -0.2
+    ret = mupdf.fz_font_descender(font)
+    return ret
+
+
+def JM_is_word_delimiter(ch, delimiters):
+    """Check if ch is an extra word delimiting character.
+    """
+    if (0
+        or ch <= 32
+        or ch == 160
+        or 0x202a <= ch <= 0x202e
+    ):
+        # covers any whitespace plus unicodes that switch between
+        # right-to-left and left-to-right languages
+        return True
+    if not delimiters:  # no extra delimiters provided
+        return False
+    char = chr(ch)
+    for d in delimiters:
+        if d == char:
+            return True
+    return False
+    
+
+def JM_is_rtl_char(ch):
+    if ch < 0x590 or ch > 0x900:
+        return False
+    return True
+
+
+def JM_font_name(font):
+    assert isinstance(font, mupdf.FzFont)
+    name = mupdf.fz_font_name(font)
+    s = name.find('+')
+    if _globals.subset_fontnames or s == -1 or s != 6:
+        return name
+    return name[s + 1:]
+
+
+def JM_gather_fonts(pdf, dict_, fontlist, stream_xref):
+    rc = 1
+    n = mupdf.pdf_dict_len(dict_)
+    for i in range(n):
+
+        refname = mupdf.pdf_dict_get_key(dict_, i)
+        fontdict = mupdf.pdf_dict_get_val(dict_, i)
+        if not mupdf.pdf_is_dict(fontdict):
+            mupdf.fz_warn( f"'{mupdf.pdf_to_name(refname)}' is no font dict ({mupdf.pdf_to_num(fontdict)} 0 R)")
+            continue
+
+        subtype = mupdf.pdf_dict_get(fontdict, mupdf.PDF_ENUM_NAME_Subtype)
+        basefont = mupdf.pdf_dict_get(fontdict, mupdf.PDF_ENUM_NAME_BaseFont)
+        if not basefont.m_internal or mupdf.pdf_is_null(basefont):
+            name = mupdf.pdf_dict_get(fontdict, mupdf.PDF_ENUM_NAME_Name)
+        else:
+            name = basefont
+        encoding = mupdf.pdf_dict_get(fontdict, mupdf.PDF_ENUM_NAME_Encoding)
+        if mupdf.pdf_is_dict(encoding):
+            encoding = mupdf.pdf_dict_get(encoding, mupdf.PDF_ENUM_NAME_BaseEncoding)
+        xref = mupdf.pdf_to_num(fontdict)
+        ext = "n/a"
+        if xref:
+            ext = JM_get_fontextension(pdf, xref)
+        entry = (
+                xref,
+                ext,
+                mupdf.pdf_to_name(subtype),
+                JM_EscapeStrFromStr(mupdf.pdf_to_name(name)),
+                mupdf.pdf_to_name(refname),
+                mupdf.pdf_to_name(encoding),
+                stream_xref,
+                )
+        fontlist.append(entry)
+    return rc
+
+
+def JM_gather_forms(doc, dict_: mupdf.PdfObj, imagelist, stream_xref: int):
+    '''
+    Store info of a /Form xobject in Python list
+    '''
+    assert isinstance(doc, mupdf.PdfDocument)
+    rc = 1
+    n = mupdf.pdf_dict_len(dict_)
+    for i in range(n):
+        refname = mupdf.pdf_dict_get_key( dict_, i)
+        imagedict = mupdf.pdf_dict_get_val(dict_, i)
+        if not mupdf.pdf_is_dict(imagedict):
+            mupdf.fz_warn( f"'{mupdf.pdf_to_name(refname)}' is no form dict ({mupdf.pdf_to_num(imagedict)} 0 R)")
+            continue
+
+        type_ = mupdf.pdf_dict_get(imagedict, PDF_NAME('Subtype'))
+        if not mupdf.pdf_name_eq(type_, PDF_NAME('Form')):
+            continue
+
+        o = mupdf.pdf_dict_get(imagedict, PDF_NAME('BBox'))
+        m = mupdf.pdf_dict_get(imagedict, PDF_NAME('Matrix'))
+        if m.m_internal:
+            mat = mupdf.pdf_to_matrix(m)
+        else:
+            mat = mupdf.FzMatrix()
+        if o.m_internal:
+            bbox = mupdf.fz_transform_rect( mupdf.pdf_to_rect(o), mat)
+        else:
+            bbox = mupdf.FzRect(mupdf.FzRect.Fixed_INFINITE)
+        xref = mupdf.pdf_to_num(imagedict)
+
+        entry = (
+                xref,
+                mupdf.pdf_to_name( refname),
+                stream_xref,
+                JM_py_from_rect(bbox),
+                )
+        imagelist.append(entry)
+    return rc
+
+
+def JM_gather_images(doc: mupdf.PdfDocument, dict_: mupdf.PdfObj, imagelist, stream_xref: int):
+    '''
+    Store info of an image in Python list
+    '''
+    rc = 1
+    n = mupdf.pdf_dict_len( dict_)
+    for i in range(n):
+        refname = mupdf.pdf_dict_get_key(dict_, i)
+        imagedict = mupdf.pdf_dict_get_val(dict_, i)
+        if not mupdf.pdf_is_dict(imagedict):
+            mupdf.fz_warn(f"'{mupdf.pdf_to_name(refname)}' is no image dict ({mupdf.pdf_to_num(imagedict)} 0 R)")
+            continue
+
+        type_ = mupdf.pdf_dict_get(imagedict, PDF_NAME('Subtype'))
+        if not mupdf.pdf_name_eq(type_, PDF_NAME('Image')):
+            continue
+
+        xref = mupdf.pdf_to_num(imagedict)
+        gen = 0
+        smask = mupdf.pdf_dict_geta(imagedict, PDF_NAME('SMask'), PDF_NAME('Mask'))
+        if smask.m_internal:
+            gen = mupdf.pdf_to_num(smask)
+
+        filter_ = mupdf.pdf_dict_geta(imagedict, PDF_NAME('Filter'), PDF_NAME('F'))
+        if mupdf.pdf_is_array(filter_):
+            filter_ = mupdf.pdf_array_get(filter_, 0)
+
+        altcs = mupdf.PdfObj(0)
+        cs = mupdf.pdf_dict_geta(imagedict, PDF_NAME('ColorSpace'), PDF_NAME('CS'))
+        if mupdf.pdf_is_array(cs):
+            cses = cs
+            cs = mupdf.pdf_array_get(cses, 0)
+            if (mupdf.pdf_name_eq(cs, PDF_NAME('DeviceN'))
+                    or mupdf.pdf_name_eq(cs, PDF_NAME('Separation'))
+                    ):
+                altcs = mupdf.pdf_array_get(cses, 2)
+                if mupdf.pdf_is_array(altcs):
+                    altcs = mupdf.pdf_array_get(altcs, 0)
+        width = mupdf.pdf_dict_geta(imagedict, PDF_NAME('Width'), PDF_NAME('W'))
+        height = mupdf.pdf_dict_geta(imagedict, PDF_NAME('Height'), PDF_NAME('H'))
+        bpc = mupdf.pdf_dict_geta(imagedict, PDF_NAME('BitsPerComponent'), PDF_NAME('BPC'))
+
+        entry = (
+                xref,
+                gen,
+                mupdf.pdf_to_int(width),
+                mupdf.pdf_to_int(height),
+                mupdf.pdf_to_int(bpc),
+                JM_EscapeStrFromStr(mupdf.pdf_to_name(cs)),
+                JM_EscapeStrFromStr(mupdf.pdf_to_name(altcs)),
+                JM_EscapeStrFromStr(mupdf.pdf_to_name(refname)),
+                JM_EscapeStrFromStr(mupdf.pdf_to_name(filter_)),
+                stream_xref,
+                )
+        imagelist.append(entry)
+    return rc
+
+
+def JM_get_annot_by_xref(page, xref):
+    '''
+    retrieve annot by its xref
+    '''
+    assert isinstance(page, mupdf.PdfPage)
+    found = 0
+    # loop thru MuPDF's internal annots array
+    annot = mupdf.pdf_first_annot(page)
+    while 1:
+        if not annot.m_internal:
+            break
+        if xref == mupdf.pdf_to_num(mupdf.pdf_annot_obj(annot)):
+            found = 1
+            break
+        annot = mupdf.pdf_next_annot( annot)
+    if not found:
+        raise Exception("xref %d is not an annot of this page" % xref)
+    return annot
+
+
+def JM_get_annot_by_name(page, name):
+    '''
+    retrieve annot by name (/NM key)
+    '''
+    assert isinstance(page, mupdf.PdfPage)
+    if not name:
+        return
+    found = 0
+    # loop thru MuPDF's internal annots and widget arrays
+    annot = mupdf.pdf_first_annot(page)
+    while 1:
+        if not annot.m_internal:
+            break
+
+        response, len_ = mupdf.pdf_to_string(mupdf.pdf_dict_gets(mupdf.pdf_annot_obj(annot), "NM"))
+        if name == response:
+            found = 1
+            break
+        annot = mupdf.pdf_next_annot(annot)
+    if not found:
+        raise Exception("'%s' is not an annot of this page" % name)
+    return annot
+
+
+def JM_get_annot_id_list(page):
+    names = []
+    annots = mupdf.pdf_dict_get( page.obj(), mupdf.PDF_ENUM_NAME_Annots)
+    if not annots.m_internal:
+        return names
+    for i in range( mupdf.pdf_array_len(annots)):
+        annot_obj = mupdf.pdf_array_get(annots, i)
+        name = mupdf.pdf_dict_gets(annot_obj, "NM")
+        if name.m_internal:
+            names.append(
+                mupdf.pdf_to_text_string(name)
+                )
+    return names
+
+def JM_get_annot_xref_list( page_obj):
+    '''
+    return the xrefs and /NM ids of a page's annots, links and fields
+    '''
+    if g_use_extra:
+        names = extra.JM_get_annot_xref_list( page_obj)
+        return names
+    
+    names = []
+    annots = mupdf.pdf_dict_get( page_obj, PDF_NAME('Annots'))
+    n = mupdf.pdf_array_len( annots)
+    for i in range( n):
+        annot_obj = mupdf.pdf_array_get( annots, i)
+        xref = mupdf.pdf_to_num( annot_obj)
+        subtype = mupdf.pdf_dict_get( annot_obj, PDF_NAME('Subtype'))
+        if not subtype.m_internal:
+            continue    # subtype is required
+        type_ = mupdf.pdf_annot_type_from_string( mupdf.pdf_to_name( subtype))
+        if type_ == mupdf.PDF_ANNOT_UNKNOWN:
+            continue    # only accept valid annot types
+        id_ = mupdf.pdf_dict_gets( annot_obj, "NM")
+        names.append( (xref, type_, mupdf.pdf_to_text_string( id_)))
+    return names
+
+
+def JM_get_annot_xref_list2(page):
+    page = page._pdf_page(required=False)
+    if not page.m_internal:
+        return list()
+    return JM_get_annot_xref_list( page.obj())
+
+
+def JM_get_border_style(style):
+    '''
+    return pdf_obj "border style" from Python str
+    '''
+    val = mupdf.PDF_ENUM_NAME_S
+    if style is None:
+        return val
+    s = style
+    if   s.startswith("b") or s.startswith("B"):    val = mupdf.PDF_ENUM_NAME_B
+    elif s.startswith("d") or s.startswith("D"):    val = mupdf.PDF_ENUM_NAME_D
+    elif s.startswith("i") or s.startswith("I"):    val = mupdf.PDF_ENUM_NAME_I
+    elif s.startswith("u") or s.startswith("U"):    val = mupdf.PDF_ENUM_NAME_U
+    elif s.startswith("s") or s.startswith("S"):    val = mupdf.PDF_ENUM_NAME_S
+    return val
+
+
+def JM_get_font(
+        fontname,
+        fontfile,
+        fontbuffer,
+        script,
+        lang,
+        ordering,
+        is_bold,
+        is_italic,
+        is_serif,
+        embed,
+        ):
+    '''
+    return a fz_font from a number of parameters
+    '''
+    def fertig(font):
+        if not font.m_internal:
+            raise RuntimeError(MSG_FONT_FAILED)
+        # if font allows this, set embedding
+        if not font.m_internal.flags.never_embed:
+            mupdf.fz_set_font_embedding(font, embed)
+        return font
+    
+    index = 0
+    font = None
+    if fontfile:
+        #goto have_file;
+        font = mupdf.fz_new_font_from_file( None, fontfile, index, 0)
+        return fertig(font)
+
+    if fontbuffer:
+        #goto have_buffer;
+        res = JM_BufferFromBytes(fontbuffer)
+        font = mupdf.fz_new_font_from_buffer( None, res, index, 0)
+        return fertig(font)
+
+    if ordering > -1:
+        # goto have_cjk;
+        font = mupdf.fz_new_cjk_font(ordering)
+        return fertig(font)
+
+    if fontname:
+        # goto have_base14;
+        # Base-14 or a MuPDF builtin font
+        font = mupdf.fz_new_base14_font(fontname)
+        if font.m_internal:
+            return fertig(font)
+        font = mupdf.fz_new_builtin_font(fontname, is_bold, is_italic)
+        return fertig(font)
+    
+    # Check for NOTO font
+    #have_noto:;
+    data, size, index = mupdf.fz_lookup_noto_font( script, lang)
+    font = None
+    if data:
+        font = mupdf.fz_new_font_from_memory( None, data, size, index, 0)
+    if font.m_internal:
+        return fertig(font)
+    font = mupdf.fz_load_fallback_font( script, lang, is_serif, is_bold, is_italic)
+    return fertig(font)
+    
+
+def JM_get_fontbuffer(doc, xref):
+    '''
+    Return the contents of a font file, identified by xref
+    '''
+    if xref < 1:
+        return
+    o = mupdf.pdf_load_object(doc, xref)
+    desft = mupdf.pdf_dict_get(o, PDF_NAME('DescendantFonts'))
+    if desft.m_internal:
+        obj = mupdf.pdf_resolve_indirect(mupdf.pdf_array_get(desft, 0))
+        obj = mupdf.pdf_dict_get(obj, PDF_NAME('FontDescriptor'))
+    else:
+        obj = mupdf.pdf_dict_get(o, PDF_NAME('FontDescriptor'))
+
+    if not obj.m_internal:
+        message(f"invalid font - FontDescriptor missing")
+        return
+
+    o = obj
+
+    stream = None
+
+    obj = mupdf.pdf_dict_get(o, PDF_NAME('FontFile'))
+    if obj.m_internal:
+        stream = obj    # ext = "pfa"
+
+    obj = mupdf.pdf_dict_get(o, PDF_NAME('FontFile2'))
+    if obj.m_internal:
+        stream = obj    # ext = "ttf"
+
+    obj = mupdf.pdf_dict_get(o, PDF_NAME('FontFile3'))
+    if obj.m_internal:
+        stream = obj
+
+        obj = mupdf.pdf_dict_get(obj, PDF_NAME('Subtype'))
+        if obj.m_internal and not mupdf.pdf_is_name(obj):
+            message("invalid font descriptor subtype")
+            return
+
+        if mupdf.pdf_name_eq(obj, PDF_NAME('Type1C')):
+            pass    # Prev code did: ext = "cff", but this has no effect.
+        elif mupdf.pdf_name_eq(obj, PDF_NAME('CIDFontType0C')):
+            pass    # Prev code did: ext = "cid", but this has no effect.
+        elif mupdf.pdf_name_eq(obj, PDF_NAME('OpenType')):
+            pass    # Prev code did: ext = "otf", but this has no effect. */
+        else:
+            message('warning: unhandled font type {pdf_to_name(ctx, obj)!r}')
+
+    if not stream:
+        message('warning: unhandled font type')
+        return
+
+    return mupdf.pdf_load_stream(stream)
+
+
+def JM_get_resource_properties(ref):
+    '''
+    Return the items of Resources/Properties (used for Marked Content)
+    Argument may be e.g. a page object or a Form XObject
+    '''
+    properties = mupdf.pdf_dict_getl(ref, PDF_NAME('Resources'), PDF_NAME('Properties'))
+    if not properties.m_internal:
+        return ()
+    else:
+        n = mupdf.pdf_dict_len(properties)
+        if n < 1:
+            return ()
+        rc = []
+        for i in range(n):
+            key = mupdf.pdf_dict_get_key(properties, i)
+            val = mupdf.pdf_dict_get_val(properties, i)
+            c = mupdf.pdf_to_name(key)
+            xref = mupdf.pdf_to_num(val)
+            rc.append((c, xref))
+    return rc
+
+
+def JM_get_widget_by_xref( page, xref):
+    '''
+    retrieve widget by its xref
+    '''
+    found = False
+    annot = mupdf.pdf_first_widget( page)
+    while annot.m_internal:
+        annot_obj = mupdf.pdf_annot_obj( annot)
+        if xref == mupdf.pdf_to_num( annot_obj):
+            found = True
+            break
+        annot = mupdf.pdf_next_widget( annot)
+    if not found:
+        raise Exception( f"xref {xref} is not a widget of this page")
+    return Annot( annot)
+
+
+def JM_get_widget_properties(annot, Widget):
+    '''
+    Populate a Python Widget object with the values from a PDF form field.
+    Called by "Page.first_widget" and "Widget.next".
+    '''
+    #log( '{type(annot)=}')
+    annot_obj = mupdf.pdf_annot_obj(annot.this)
+    #log( 'Have called mupdf.pdf_annot_obj()')
+    page = _pdf_annot_page(annot.this)
+    pdf = page.doc()
+    tw = annot
+
+    def SETATTR(key, value):
+        setattr(Widget, key, value)
+
+    def SETATTR_DROP(mod, key, value):
+        # Original C code for this function deletes if PyObject* is NULL. We
+        # don't have a representation for that in Python - e.g. None is not
+        # represented by NULL.
+        setattr(mod, key, value)
+
+    #log( '=== + mupdf.pdf_widget_type(tw)')
+    field_type = mupdf.pdf_widget_type(tw.this)
+    #log( '=== - mupdf.pdf_widget_type(tw)')
+    Widget.field_type = field_type
+    if field_type == mupdf.PDF_WIDGET_TYPE_SIGNATURE:
+        if mupdf.pdf_signature_is_signed(pdf, annot_obj):
+            SETATTR("is_signed", True)
+        else:
+            SETATTR("is_signed",False)
+    else:
+        SETATTR("is_signed", None)
+    SETATTR_DROP(Widget, "border_style", JM_UnicodeFromStr(mupdf.pdf_field_border_style(annot_obj)))
+    SETATTR_DROP(Widget, "field_type_string", JM_UnicodeFromStr(JM_field_type_text(field_type)))
+
+    field_name = mupdf.pdf_load_field_name(annot_obj)
+    SETATTR_DROP(Widget, "field_name", field_name)
+
+    def pdf_dict_get_inheritable_nonempty_label(node, key):
+        '''
+        This is a modified version of MuPDF's pdf_dict_get_inheritable(), with
+        some changes:
+        * Returns string from pdf_to_text_string() or None if not found.
+        * Recurses to parent if current node exists but with empty string
+          value.
+        '''
+        slow = node
+        halfbeat = 11   # Don't start moving slow pointer for a while.
+        while 1:
+            if not node.m_internal:
+                return
+            val = mupdf.pdf_dict_get(node, key)
+            if val.m_internal:
+                label = mupdf.pdf_to_text_string(val)
+                if label:
+                    return label
+            node = mupdf.pdf_dict_get(node, PDF_NAME('Parent'))
+            if node.m_internal == slow.m_internal:
+                raise Exception("cycle in resources")
+            halfbeat -= 1
+            if halfbeat == 0:
+                slow = mupdf.pdf_dict_get(slow, PDF_NAME('Parent'))
+                halfbeat = 2
+    
+    # In order to address #3950, we use our modified pdf_dict_get_inheritable()
+    # to ignore empty-string child values.
+    label = pdf_dict_get_inheritable_nonempty_label(annot_obj, PDF_NAME('TU'))
+    if label is not None:
+        SETATTR_DROP(Widget, "field_label", label)
+
+    fvalue = None
+    if field_type == mupdf.PDF_WIDGET_TYPE_RADIOBUTTON:
+        obj = mupdf.pdf_dict_get( annot_obj, PDF_NAME('Parent'))    # owning RB group
+        if obj.m_internal:
+            SETATTR_DROP(Widget, "rb_parent", mupdf.pdf_to_num( obj))
+        obj = mupdf.pdf_dict_get(annot_obj, PDF_NAME('AS'))
+        if obj.m_internal:
+            fvalue = mupdf.pdf_to_name(obj)
+    if not fvalue:
+        fvalue = mupdf.pdf_field_value(annot_obj)
+    SETATTR_DROP(Widget, "field_value", JM_UnicodeFromStr(fvalue))
+
+    SETATTR_DROP(Widget, "field_display", mupdf.pdf_field_display(annot_obj))
+
+    border_width = mupdf.pdf_to_real(mupdf.pdf_dict_getl(annot_obj, PDF_NAME('BS'), PDF_NAME('W')))
+    if border_width == 0:
+        border_width = 1
+    SETATTR_DROP(Widget, "border_width", border_width)
+
+    obj = mupdf.pdf_dict_getl(annot_obj, PDF_NAME('BS'), PDF_NAME('D'))
+    if mupdf.pdf_is_array(obj):
+        n = mupdf.pdf_array_len(obj)
+        d = [0] * n
+        for i in range(n):
+            d[i] = mupdf.pdf_to_int(mupdf.pdf_array_get(obj, i))
+        SETATTR_DROP(Widget, "border_dashes", d)
+
+    SETATTR_DROP(Widget, "text_maxlen", mupdf.pdf_text_widget_max_len(tw.this))
+
+    SETATTR_DROP(Widget, "text_format", mupdf.pdf_text_widget_format(tw.this))
+
+    obj = mupdf.pdf_dict_getl(annot_obj, PDF_NAME('MK'), PDF_NAME('BG'))
+    if mupdf.pdf_is_array(obj):
+        n = mupdf.pdf_array_len(obj)
+        col = [0] * n
+        for i in range(n):
+            col[i] = mupdf.pdf_to_real(mupdf.pdf_array_get(obj, i))
+        SETATTR_DROP(Widget, "fill_color", col)
+
+    obj = mupdf.pdf_dict_getl(annot_obj, PDF_NAME('MK'), PDF_NAME('BC'))
+    if mupdf.pdf_is_array(obj):
+        n = mupdf.pdf_array_len(obj)
+        col = [0] * n
+        for i in range(n):
+            col[i] = mupdf.pdf_to_real(mupdf.pdf_array_get(obj, i))
+        SETATTR_DROP(Widget, "border_color", col)
+
+    SETATTR_DROP(Widget, "choice_values", JM_choice_options(annot))
+
+    da = mupdf.pdf_to_text_string(mupdf.pdf_dict_get_inheritable(annot_obj, PDF_NAME('DA')))
+    SETATTR_DROP(Widget, "_text_da", JM_UnicodeFromStr(da))
+
+    obj = mupdf.pdf_dict_getl(annot_obj, PDF_NAME('MK'), PDF_NAME('CA'))
+    if obj.m_internal:
+        SETATTR_DROP(Widget, "button_caption", JM_UnicodeFromStr(mupdf.pdf_to_text_string(obj)))
+
+    SETATTR_DROP(Widget, "field_flags", mupdf.pdf_field_flags(annot_obj))
+
+    # call Py method to reconstruct text color, font name, size
+    Widget._parse_da()
+
+    # extract JavaScript action texts
+    s = mupdf.pdf_dict_get(annot_obj, PDF_NAME('A'))
+    ss = JM_get_script(s)
+    SETATTR_DROP(Widget, "script", ss)
+
+    SETATTR_DROP(Widget, "script_stroke",
+            JM_get_script(mupdf.pdf_dict_getl(annot_obj, PDF_NAME('AA'), PDF_NAME('K')))
+            )
+
+    SETATTR_DROP(Widget, "script_format",
+            JM_get_script(mupdf.pdf_dict_getl(annot_obj, PDF_NAME('AA'), PDF_NAME('F')))
+            )
+
+    SETATTR_DROP(Widget, "script_change",
+            JM_get_script(mupdf.pdf_dict_getl(annot_obj, PDF_NAME('AA'), PDF_NAME('V')))
+            )
+
+    SETATTR_DROP(Widget, "script_calc",
+            JM_get_script(mupdf.pdf_dict_getl(annot_obj, PDF_NAME('AA'), PDF_NAME('C')))
+            )
+
+    SETATTR_DROP(Widget, "script_blur",
+            JM_get_script(mupdf.pdf_dict_getl(annot_obj, PDF_NAME('AA'), mupdf.pdf_new_name('Bl')))
+            )
+
+    SETATTR_DROP(Widget, "script_focus",
+            JM_get_script(mupdf.pdf_dict_getl(annot_obj, PDF_NAME('AA'), mupdf.pdf_new_name('Fo')))
+            )
+
+
+def JM_get_fontextension(doc, xref):
+    '''
+    Return the file extension of a font file, identified by xref
+    '''
+    if xref < 1:
+        return "n/a"
+    o = mupdf.pdf_load_object(doc, xref)
+    desft = mupdf.pdf_dict_get(o, PDF_NAME('DescendantFonts'))
+    if desft.m_internal:
+        obj = mupdf.pdf_resolve_indirect(mupdf.pdf_array_get(desft, 0))
+        obj = mupdf.pdf_dict_get(obj, PDF_NAME('FontDescriptor'))
+    else:
+        obj = mupdf.pdf_dict_get(o, PDF_NAME('FontDescriptor'))
+    if not obj.m_internal:
+        return "n/a"    # this is a base-14 font
+
+    o = obj # we have the FontDescriptor
+
+    obj = mupdf.pdf_dict_get(o, PDF_NAME('FontFile'))
+    if obj.m_internal:
+        return "pfa"
+
+    obj = mupdf.pdf_dict_get(o, PDF_NAME('FontFile2'))
+    if obj.m_internal:
+        return "ttf"
+
+    obj = mupdf.pdf_dict_get(o, PDF_NAME('FontFile3'))
+    if obj.m_internal:
+        obj = mupdf.pdf_dict_get(obj, PDF_NAME('Subtype'))
+        if obj.m_internal and not mupdf.pdf_is_name(obj):
+            message("invalid font descriptor subtype")
+            return "n/a"
+        if mupdf.pdf_name_eq(obj, PDF_NAME('Type1C')):
+            return "cff"
+        elif mupdf.pdf_name_eq(obj, PDF_NAME('CIDFontType0C')):
+            return "cid"
+        elif mupdf.pdf_name_eq(obj, PDF_NAME('OpenType')):
+            return "otf"
+        else:
+            message("unhandled font type '%s'", mupdf.pdf_to_name(obj))
+
+    return "n/a"
+
+
+def JM_get_ocg_arrays_imp(arr):
+    '''
+    Get OCG arrays from OC configuration
+    Returns dict {"basestate":name, "on":list, "off":list, "rbg":list, "locked":list}
+    '''
+    list_ = list()
+    if mupdf.pdf_is_array( arr):
+        n = mupdf.pdf_array_len( arr)
+        for i in range(n):
+            obj = mupdf.pdf_array_get( arr, i)
+            item = mupdf.pdf_to_num( obj)
+            if item not in list_:
+                list_.append(item)
+    return list_
+
+
+def JM_get_ocg_arrays(conf):
+
+    rc = dict()
+    arr = mupdf.pdf_dict_get( conf, PDF_NAME('ON'))
+    list_ = JM_get_ocg_arrays_imp( arr)
+    if list_:
+        rc["on"] = list_
+    arr = mupdf.pdf_dict_get( conf, PDF_NAME('OFF'))
+    list_ = JM_get_ocg_arrays_imp( arr)
+    if list_:
+        rc["off"] = list_
+    arr = mupdf.pdf_dict_get( conf, PDF_NAME('Locked'))
+    list_ = JM_get_ocg_arrays_imp( arr)
+    if list_:
+        rc['locked'] = list_
+    list_ = list()
+    arr = mupdf.pdf_dict_get( conf, PDF_NAME('RBGroups'))
+    if mupdf.pdf_is_array( arr):
+        n = mupdf.pdf_array_len( arr)
+        for i in range(n):
+            obj = mupdf.pdf_array_get( arr, i)
+            list1 = JM_get_ocg_arrays_imp( obj)
+            list_.append(list1)
+    if list_:
+        rc["rbgroups"] = list_
+    obj = mupdf.pdf_dict_get( conf, PDF_NAME('BaseState'))
+
+    if obj.m_internal:
+        state = mupdf.pdf_to_name( obj)
+        rc["basestate"] = state
+    return rc
+
+
+def JM_get_page_labels(liste, nums):
+    n = mupdf.pdf_array_len(nums)
+    for i in range(0, n, 2):
+        key = mupdf.pdf_resolve_indirect( mupdf.pdf_array_get(nums, i))
+        pno = mupdf.pdf_to_int(key)
+        val = mupdf.pdf_resolve_indirect( mupdf.pdf_array_get(nums, i + 1))
+        res = JM_object_to_buffer(val, 1, 0)
+        c = mupdf.fz_buffer_extract(res)
+        assert isinstance(c, bytes)
+        c = c.decode('utf-8')
+        liste.append( (pno, c))
+
+
+def JM_get_script(key):
+    '''
+    JavaScript extractor
+    Returns either the script source or None. Parameter is a PDF action
+    dictionary, which must have keys /S and /JS. The value of /S must be
+    '/JavaScript'. The value of /JS is returned.
+    '''
+    if not key.m_internal:
+        return
+
+    j = mupdf.pdf_dict_get(key, PDF_NAME('S'))
+    jj = mupdf.pdf_to_name(j)
+    if jj == "JavaScript":
+        js = mupdf.pdf_dict_get(key, PDF_NAME('JS'))
+        if not js.m_internal:
+            return
+    else:
+        return
+
+    if mupdf.pdf_is_string(js):
+        script = JM_UnicodeFromStr(mupdf.pdf_to_text_string(js))
+    elif mupdf.pdf_is_stream(js):
+        res = mupdf.pdf_load_stream(js)
+        script = JM_EscapeStrFromBuffer(res)
+    else:
+        return
+    if script:  # do not return an empty script
+        return script
+    return
+
+
+def JM_have_operation(pdf):
+    '''
+    Ensure valid journalling state
+    '''
+    if pdf.m_internal.journal and not mupdf.pdf_undoredo_step(pdf, 0):
+        return 0
+    return 1
+
+
+def JM_image_extension(type_):
+    '''
+    return extension for MuPDF image type
+    '''
+    if type_ == mupdf.FZ_IMAGE_FAX:     return "fax"
+    if type_ == mupdf.FZ_IMAGE_RAW:     return "raw"
+    if type_ == mupdf.FZ_IMAGE_FLATE:   return "flate"
+    if type_ == mupdf.FZ_IMAGE_LZW:     return "lzw"
+    if type_ == mupdf.FZ_IMAGE_RLD:     return "rld"
+    if type_ == mupdf.FZ_IMAGE_BMP:     return "bmp"
+    if type_ == mupdf.FZ_IMAGE_GIF:     return "gif"
+    if type_ == mupdf.FZ_IMAGE_JBIG2:   return "jb2"
+    if type_ == mupdf.FZ_IMAGE_JPEG:    return "jpeg"
+    if type_ == mupdf.FZ_IMAGE_JPX:     return "jpx"
+    if type_ == mupdf.FZ_IMAGE_JXR:     return "jxr"
+    if type_ == mupdf.FZ_IMAGE_PNG:     return "png"
+    if type_ == mupdf.FZ_IMAGE_PNM:     return "pnm"
+    if type_ == mupdf.FZ_IMAGE_TIFF:    return "tiff"
+    #if type_ == mupdf.FZ_IMAGE_PSD:     return "psd"
+    return "n/a"
+
+
+# fixme: need to avoid using a global for this.
+g_img_info = None
+
+
+def JM_image_filter(opaque, ctm, name, image):
+    assert isinstance(ctm, mupdf.FzMatrix)
+    r = mupdf.FzRect(mupdf.FzRect.Fixed_UNIT)
+    q = mupdf.fz_transform_quad( mupdf.fz_quad_from_rect(r), ctm)
+    q = mupdf.fz_transform_quad( q, g_img_info_matrix)
+    temp = name, JM_py_from_quad(q)
+    g_img_info.append(temp)
+
+
+def JM_image_profile( imagedata, keep_image):
+    '''
+    Return basic properties of an image provided as bytes or bytearray
+    The function creates an fz_image and optionally returns it.
+    '''
+    if not imagedata:
+        return None # nothing given
+    
+    len_ = len( imagedata)
+    if len_ < 8:
+        message( "bad image data")
+        return None
+    c = imagedata
+    #log( 'calling mfz_recognize_image_format with {c!r=}')
+    type_ = mupdf.fz_recognize_image_format( c)
+    if type_ == mupdf.FZ_IMAGE_UNKNOWN:
+        return None
+
+    if keep_image:
+        res = mupdf.fz_new_buffer_from_copied_data( c, len_)
+    else:
+        res = mupdf.fz_new_buffer_from_shared_data( c, len_)
+    image = mupdf.fz_new_image_from_buffer( res)
+    ctm = mupdf.fz_image_orientation_matrix( image)
+    xres, yres = mupdf.fz_image_resolution(image)
+    orientation = mupdf.fz_image_orientation( image)
+    cs_name = mupdf.fz_colorspace_name( image.colorspace())
+    result = dict()
+    result[ dictkey_width] = image.w()
+    result[ dictkey_height] = image.h()
+    result[ "orientation"] = orientation
+    result[ dictkey_matrix] = JM_py_from_matrix(ctm)
+    result[ dictkey_xres] = xres
+    result[ dictkey_yres] = yres
+    result[ dictkey_colorspace] = image.n()
+    result[ dictkey_bpc] = image.bpc()
+    result[ dictkey_ext] = JM_image_extension(type_)
+    result[ dictkey_cs_name] = cs_name
+
+    if keep_image:
+        result[ dictkey_image] = image
+    return result
+
+
+def JM_image_reporter(page):
+    doc = page.doc()
+    global g_img_info_matrix
+    g_img_info_matrix = mupdf.FzMatrix()
+    mediabox = mupdf.FzRect()
+    mupdf.pdf_page_transform(page, mediabox, g_img_info_matrix)
+
+    class SanitizeFilterOptions(mupdf.PdfSanitizeFilterOptions2):
+        def __init__(self):
+            super().__init__()
+            self.use_virtual_image_filter()
+        def image_filter(self, ctx, ctm, name, image, scissor):
+            JM_image_filter(None, mupdf.FzMatrix(ctm), name, image)
+
+    sanitize_filter_options = SanitizeFilterOptions()
+
+    filter_options = _make_PdfFilterOptions(
+            instance_forms=1,
+            ascii=1,
+            no_update=1,
+            sanitize=1,
+            sopts=sanitize_filter_options,
+            )
+
+    global g_img_info
+    g_img_info = []
+
+    mupdf.pdf_filter_page_contents( doc, page, filter_options)
+
+    rc = tuple(g_img_info)
+    g_img_info = []
+    return rc
+
+
+def JM_fitz_config():
+    have_TOFU           = not hasattr(mupdf, 'TOFU')
+    have_TOFU_BASE14    = not hasattr(mupdf, 'TOFU_BASE14')
+    have_TOFU_CJK       = not hasattr(mupdf, 'TOFU_CJK')
+    have_TOFU_CJK_EXT   = not hasattr(mupdf, 'TOFU_CJK_EXT')
+    have_TOFU_CJK_LANG  = not hasattr(mupdf, 'TOFU_CJK_LANG')
+    have_TOFU_EMOJI     = not hasattr(mupdf, 'TOFU_EMOJI')
+    have_TOFU_HISTORIC  = not hasattr(mupdf, 'TOFU_HISTORIC')
+    have_TOFU_SIL       = not hasattr(mupdf, 'TOFU_SIL')
+    have_TOFU_SYMBOL    = not hasattr(mupdf, 'TOFU_SYMBOL')
+    
+    ret = dict()
+    ret["base14"]           = have_TOFU_BASE14
+    ret["cbz"]              = bool(mupdf.FZ_ENABLE_CBZ)
+    ret["epub"]             = bool(mupdf.FZ_ENABLE_EPUB)
+    ret["html"]             = bool(mupdf.FZ_ENABLE_HTML)
+    ret["icc"]              = bool(mupdf.FZ_ENABLE_ICC)
+    ret["img"]              = bool(mupdf.FZ_ENABLE_IMG)
+    ret["jpx"]              = bool(mupdf.FZ_ENABLE_JPX)
+    ret["js"]               = bool(mupdf.FZ_ENABLE_JS)
+    ret["pdf"]              = bool(mupdf.FZ_ENABLE_PDF)
+    ret["plotter-cmyk"]     = bool(mupdf.FZ_PLOTTERS_CMYK)
+    ret["plotter-g"]        = bool(mupdf.FZ_PLOTTERS_G)
+    ret["plotter-n"]        = bool(mupdf.FZ_PLOTTERS_N)
+    ret["plotter-rgb"]      = bool(mupdf.FZ_PLOTTERS_RGB)
+    ret["py-memory"]        = bool(JM_MEMORY)
+    ret["svg"]              = bool(mupdf.FZ_ENABLE_SVG)
+    ret["tofu"]             = have_TOFU
+    ret["tofu-cjk"]         = have_TOFU_CJK
+    ret["tofu-cjk-ext"]     = have_TOFU_CJK_EXT
+    ret["tofu-cjk-lang"]    = have_TOFU_CJK_LANG
+    ret["tofu-emoji"]       = have_TOFU_EMOJI
+    ret["tofu-historic"]    = have_TOFU_HISTORIC
+    ret["tofu-sil"]         = have_TOFU_SIL
+    ret["tofu-symbol"]      = have_TOFU_SYMBOL
+    ret["xps"]              = bool(mupdf.FZ_ENABLE_XPS)
+    return ret
+
+
+def JM_insert_contents(pdf, pageref, newcont, overlay):
+    '''
+    Insert a buffer as a new separate /Contents object of a page.
+    1. Create a new stream object from buffer 'newcont'
+    2. If /Contents already is an array, then just prepend or append this object
+    3. Else, create new array and put old content obj and this object into it.
+       If the page had no /Contents before, just create a 1-item array.
+    '''
+    contents = mupdf.pdf_dict_get(pageref, PDF_NAME('Contents'))
+    newconts = mupdf.pdf_add_stream(pdf, newcont, mupdf.PdfObj(), 0)
+    xref = mupdf.pdf_to_num(newconts)
+    if mupdf.pdf_is_array(contents):
+        if overlay:  # append new object
+            mupdf.pdf_array_push(contents, newconts)
+        else:   # prepend new object
+            mupdf.pdf_array_insert(contents, newconts, 0)
+    else:
+        carr = mupdf.pdf_new_array(pdf, 5)
+        if overlay:
+            if contents.m_internal:
+                mupdf.pdf_array_push(carr, contents)
+            mupdf.pdf_array_push(carr, newconts)
+        else:
+            mupdf.pdf_array_push(carr, newconts)
+            if contents.m_internal:
+                mupdf.pdf_array_push(carr, contents)
+        mupdf.pdf_dict_put(pageref, PDF_NAME('Contents'), carr)
+    return xref
+
+
+def JM_insert_font(pdf, bfname, fontfile, fontbuffer, set_simple, idx, wmode, serif, encoding, ordering):
+    '''
+    Insert a font in a PDF
+    '''
+    font = None
+    res = None
+    data = None
+    ixref = 0
+    index = 0
+    simple = 0
+    value=None
+    name=None
+    subt=None
+    exto = None
+
+    ENSURE_OPERATION(pdf)
+    # check for CJK font
+    if ordering > -1:
+        data, size, index = mupdf.fz_lookup_cjk_font(ordering)
+    if data:
+        font = mupdf.fz_new_font_from_memory(None, data, size, index, 0)
+        font_obj = mupdf.pdf_add_cjk_font(pdf, font, ordering, wmode, serif)
+        exto = "n/a"
+        simple = 0
+        #goto weiter;
+    else:
+
+        # check for PDF Base-14 font
+        if bfname:
+            data, size = mupdf.fz_lookup_base14_font(bfname)
+        if data:
+            font = mupdf.fz_new_font_from_memory(bfname, data, size, 0, 0)
+            font_obj = mupdf.pdf_add_simple_font(pdf, font, encoding)
+            exto = "n/a"
+            simple = 1
+            #goto weiter;
+
+        else:
+            if fontfile:
+                font = mupdf.fz_new_font_from_file(None, fontfile, idx, 0)
+            else:
+                res = JM_BufferFromBytes(fontbuffer)
+                if not res.m_internal:
+                    RAISEPY(MSG_FILE_OR_BUFFER, PyExc_ValueError)
+                font = mupdf.fz_new_font_from_buffer(None, res, idx, 0)
+
+            if not set_simple:
+                font_obj = mupdf.pdf_add_cid_font(pdf, font)
+                simple = 0
+            else:
+                font_obj = mupdf.pdf_add_simple_font(pdf, font, encoding)
+                simple = 2
+    #weiter: ;
+    ixref = mupdf.pdf_to_num(font_obj)
+    name = JM_EscapeStrFromStr( mupdf.pdf_to_name( mupdf.pdf_dict_get(font_obj, PDF_NAME('BaseFont'))))
+
+    subt = JM_UnicodeFromStr( mupdf.pdf_to_name( mupdf.pdf_dict_get( font_obj, PDF_NAME('Subtype'))))
+
+    if not exto:
+        exto = JM_UnicodeFromStr(JM_get_fontextension(pdf, ixref))
+
+    asc = mupdf.fz_font_ascender(font)
+    dsc = mupdf.fz_font_descender(font)
+    value = [
+            ixref,
+            {
+                "name": name,        # base font name
+                "type": subt,        # subtype
+                "ext": exto,         # file extension
+                "simple": bool(simple), # simple font?
+                "ordering": ordering, # CJK font?
+                "ascender": asc,
+                "descender": dsc,
+            },
+            ]
+    return value
+
+def JM_irect_from_py(r):
+    '''
+    PySequence to mupdf.FzIrect. Default: infinite irect
+    '''
+    if isinstance(r, mupdf.FzIrect):
+        return r
+    if isinstance(r, IRect):
+        r = mupdf.FzIrect( r.x0, r.y0, r.x1, r.y1)
+        return r
+    if isinstance(r, Rect):
+        ret = mupdf.FzRect(r.x0, r.y0, r.x1, r.y1)
+        ret = mupdf.FzIrect(ret)  # Uses fz_irect_from_rect().
+        return ret
+    if isinstance(r, mupdf.FzRect):
+        ret = mupdf.FzIrect(r)  # Uses fz_irect_from_rect().
+        return ret
+    if not r or not PySequence_Check(r) or PySequence_Size(r) != 4:
+        return mupdf.FzIrect(mupdf.fz_infinite_irect)
+    f = [0, 0, 0, 0]
+    for i in range(4):
+        f[i] = r[i]
+        if f[i] is None:
+            return mupdf.FzIrect(mupdf.fz_infinite_irect)
+        if f[i] < FZ_MIN_INF_RECT:
+            f[i] = FZ_MIN_INF_RECT
+        if f[i] > FZ_MAX_INF_RECT:
+            f[i] = FZ_MAX_INF_RECT
+    return mupdf.fz_make_irect(f[0], f[1], f[2], f[3])
+
+def JM_listbox_value( annot):
+    '''
+    ListBox retrieve value
+    '''
+    # may be single value or array
+    annot_obj = mupdf.pdf_annot_obj( annot)
+    optarr = mupdf.pdf_dict_get( annot_obj, PDF_NAME('V'))
+    if mupdf.pdf_is_string( optarr):   # a single string
+        return mupdf.pdf_to_text_string( optarr)
+
+    # value is an array (may have len 0)
+    n = mupdf.pdf_array_len( optarr)
+    liste = []
+
+    # extract a list of strings
+    # each entry may again be an array: take second entry then
+    for i in range( n):
+        elem = mupdf.pdf_array_get( optarr, i)
+        if mupdf.pdf_is_array( elem):
+            elem = mupdf.pdf_array_get( elem, 1)
+        liste.append( JM_UnicodeFromStr( mupdf.pdf_to_text_string( elem)))
+    return liste
+
+
+def JM_make_annot_DA(annot, ncol, col, fontname, fontsize):
+    # PyMuPDF uses a fz_buffer to build up the string, but it's non-trivial to
+    # convert the fz_buffer's `unsigned char*` into a `const char*` suitable
+    # for passing to pdf_dict_put_text_string(). So instead we build up the
+    # string directly in Python.
+    buf = ''
+    if ncol < 1:
+        buf += f'0 g '
+    elif ncol == 1:
+        buf += f'{col[0]:g} g '
+    elif ncol == 2:
+        assert 0
+    elif ncol == 3:
+        buf += f'{col[0]:g} {col[1]:g} {col[2]:g} rg '
+    else:
+        buf += f'{col[0]:g} {col[1]:g} {col[2]:g} {col[3]:g} k '
+    buf += f'/{JM_expand_fname(fontname)} {fontsize} Tf'
+    mupdf.pdf_dict_put_text_string(mupdf.pdf_annot_obj(annot), mupdf.PDF_ENUM_NAME_DA, buf)
+
+
+def JM_make_spanlist(line_dict, line, raw, buff, tp_rect):
+    if g_use_extra:
+        return extra.JM_make_spanlist(line_dict, line, raw, buff, tp_rect)
+    char_list = None
+    span_list = []
+    mupdf.fz_clear_buffer(buff)
+    span_rect = mupdf.FzRect(mupdf.FzRect.Fixed_EMPTY)
+    line_rect = mupdf.FzRect(mupdf.FzRect.Fixed_EMPTY)
+
+    class char_style:
+        def __init__(self, rhs=None):
+            if rhs:
+                self.size = rhs.size
+                self.flags = rhs.flags
+                if mupdf_version_tuple >= (1, 25, 2):
+                    self.char_flags = rhs.char_flags
+                self.font = rhs.font
+                self.argb = rhs.argb
+                self.asc = rhs.asc
+                self.desc = rhs.desc
+                self.bidi = rhs.bidi
+            else:
+                self.size = -1
+                self.flags = -1
+                if mupdf_version_tuple >= (1, 25, 2):
+                    self.char_flags = -1
+                self.font = ''
+                self.argb = -1
+                self.asc = 0
+                self.desc = 0
+                self.bidi = 0
+        def __str__(self):
+            ret = f'{self.size} {self.flags}'
+            if mupdf_version_tuple >= (1, 25, 2):
+                ret += f' {self.char_flags}'
+            ret += f' {self.font} {self.color} {self.asc} {self.desc}'
+            return ret
+
+    old_style = char_style()
+    style = char_style()
+    span = None
+    span_origin = None
+
+    for ch in line:
+        # start-trace
+        r = JM_char_bbox(line, ch)
+        if (not JM_rects_overlap(tp_rect, r)
+                and not mupdf.fz_is_infinite_rect(tp_rect)
+                ):
+            continue
+
+        # Info from:
+        # detect_super_script()
+        # fz_font_is_italic()
+        # fz_font_is_serif()
+        # fz_font_is_monospaced()
+        # fz_font_is_bold()
+        
+        flags = JM_char_font_flags(mupdf.FzFont(mupdf.ll_fz_keep_font(ch.m_internal.font)), line, ch)
+        origin = mupdf.FzPoint(ch.m_internal.origin)
+        style.size = ch.m_internal.size
+        style.flags = flags
+        if mupdf_version_tuple >= (1, 25, 2):
+            # FZ_STEXT_SYNTHETIC is per-char, not per-span.
+            style.char_flags = ch.m_internal.flags & ~mupdf.FZ_STEXT_SYNTHETIC
+        style.font = JM_font_name(mupdf.FzFont(mupdf.ll_fz_keep_font(ch.m_internal.font)))
+        style.argb = ch.m_internal.argb
+        style.asc = JM_font_ascender(mupdf.FzFont(mupdf.ll_fz_keep_font(ch.m_internal.font)))
+        style.desc = JM_font_descender(mupdf.FzFont(mupdf.ll_fz_keep_font(ch.m_internal.font)))
+        style.bidi = ch.m_internal.bidi
+
+        if (style.size != old_style.size
+                or style.flags != old_style.flags
+                or (mupdf_version_tuple >= (1, 25, 2)
+                    and (style.char_flags != old_style.char_flags)
+                    )
+                or style.argb != old_style.argb
+                or style.font != old_style.font
+                or style.bidi != old_style.bidi
+                ):
+            if old_style.size >= 0:
+                # not first one, output previous
+                if raw:
+                    # put character list in the span
+                    span[dictkey_chars] = char_list
+                    char_list = None
+                else:
+                    # put text string in the span
+                    span[dictkey_text] = JM_EscapeStrFromBuffer( buff)
+                    mupdf.fz_clear_buffer(buff)
+
+                span[dictkey_origin] = JM_py_from_point(span_origin)
+                span[dictkey_bbox] = JM_py_from_rect(span_rect)
+                line_rect = mupdf.fz_union_rect(line_rect, span_rect)
+                span_list.append( span)
+                span = None
+
+            span = dict()
+            asc = style.asc
+            desc = style.desc
+            if style.asc < 1e-3:
+                asc = 0.9
+                desc = -0.1
+
+            span[dictkey_size] = style.size
+            span[dictkey_flags] = style.flags
+            span[dictkey_bidi] = style.bidi
+            if mupdf_version_tuple >= (1, 25, 2):
+                span[dictkey_char_flags] = style.char_flags
+            span[dictkey_font] = JM_EscapeStrFromStr(style.font)
+            span[dictkey_color] = style.argb & 0xffffff
+            if mupdf_version_tuple >= (1, 25, 0):
+                span['alpha'] = style.argb >> 24
+            span["ascender"] = asc
+            span["descender"] = desc
+
+            # Need to be careful here - doing 'old_style=style' does a shallow
+            # copy, but we need to keep old_style as a distinct instance.
+            old_style = char_style(style)
+            span_rect = r
+            span_origin = origin
+
+        span_rect = mupdf.fz_union_rect(span_rect, r)
+
+        if raw: # make and append a char dict
+            char_dict = dict()
+            char_dict[dictkey_origin] = JM_py_from_point( ch.m_internal.origin)
+            char_dict[dictkey_bbox] = JM_py_from_rect(r)
+            char_dict[dictkey_c] = chr(ch.m_internal.c)
+            char_dict['synthetic'] = bool(ch.m_internal.flags & mupdf.FZ_STEXT_SYNTHETIC)
+
+            if char_list is None:
+                char_list = []
+            char_list.append(char_dict)
+        else:   # add character byte to buffer
+            JM_append_rune(buff, ch.m_internal.c)
+
+    # all characters processed, now flush remaining span
+    if span:
+        if raw:
+            span[dictkey_chars] = char_list
+            char_list = None
+        else:
+            span[dictkey_text] = JM_EscapeStrFromBuffer(buff)
+            mupdf.fz_clear_buffer(buff)
+        span[dictkey_origin] = JM_py_from_point(span_origin)
+        span[dictkey_bbox] = JM_py_from_rect(span_rect)
+
+        if not mupdf.fz_is_empty_rect(span_rect):
+            span_list.append(span)
+            line_rect = mupdf.fz_union_rect(line_rect, span_rect)
+        span = None
+    if not mupdf.fz_is_empty_rect(line_rect):
+        line_dict[dictkey_spans] = span_list
+    else:
+        line_dict[dictkey_spans] = span_list
+    return line_rect
+
+def _make_image_dict(img, img_dict):
+    """Populate a dictionary with information extracted from a given image.
+
+    Used by 'Document.extract_image' and by 'JM_make_image_block'.
+    Both of these functions will add some more specific information.
+    """
+    img_type = img.fz_compressed_image_type()
+    ext = JM_image_extension(img_type)
+
+    # compressed image buffer if present, else None
+    ll_cbuf = mupdf.ll_fz_compressed_image_buffer(img.m_internal)
+
+    if (0
+        or not ll_cbuf
+        or img_type in (mupdf.FZ_IMAGE_JBIG2, mupdf.FZ_IMAGE_UNKNOWN)
+        or img_type < mupdf.FZ_IMAGE_BMP
+    ):
+        # not an image with a compressed buffer: convert to PNG
+        res = mupdf.fz_new_buffer_from_image_as_png(
+                    img,
+                    mupdf.FzColorParams(mupdf.fz_default_color_params),
+              )
+        ext = "png"
+    elif ext == "jpeg" and img.n() == 4:
+        # JPEG with CMYK: invert colors
+        res = mupdf.fz_new_buffer_from_image_as_jpeg(
+                    img, mupdf.FzColorParams(mupdf.fz_default_color_params), 95, 1)
+    else:
+        # copy the compressed buffer
+        res = mupdf.FzBuffer(mupdf.ll_fz_keep_buffer(ll_cbuf.buffer))
+
+    bytes_ = JM_BinFromBuffer(res)
+    img_dict[dictkey_width] = img.w()
+    img_dict[dictkey_height] = img.h()
+    img_dict[dictkey_ext] = ext
+    img_dict[dictkey_colorspace] = img.n()
+    img_dict[dictkey_xres] = img.xres()
+    img_dict[dictkey_yres] = img.yres()
+    img_dict[dictkey_bpc] = img.bpc()
+    img_dict[dictkey_size] = len(bytes_)
+    img_dict[dictkey_image] = bytes_
+
+def JM_make_image_block(block, block_dict):
+    img = block.i_image()
+    _make_image_dict(img, block_dict)
+    # if the image has a mask, store it as a PNG buffer
+    mask = img.mask()
+    if mask.m_internal:
+        buff = mask.fz_new_buffer_from_image_as_png(mupdf.FzColorParams(mupdf.fz_default_color_params))
+        block_dict["mask"] = buff.fz_buffer_extract()
+    else:
+        block_dict["mask"] = None
+    block_dict[dictkey_matrix] = JM_py_from_matrix(block.i_transform())
+
+
+def JM_make_text_block(block, block_dict, raw, buff, tp_rect):
+    if g_use_extra:
+        return extra.JM_make_text_block(block.m_internal, block_dict, raw, buff.m_internal, tp_rect.m_internal)
+    line_list = []
+    block_rect = mupdf.FzRect(mupdf.FzRect.Fixed_EMPTY)
+    #log(f'{block=}')
+    for line in block:
+        #log(f'{line=}')
+        if (mupdf.fz_is_empty_rect(mupdf.fz_intersect_rect(tp_rect, mupdf.FzRect(line.m_internal.bbox)))
+                and not mupdf.fz_is_infinite_rect(tp_rect)
+                ):
+            continue
+        line_dict = dict()
+        line_rect = JM_make_spanlist(line_dict, line, raw, buff, tp_rect)
+        block_rect = mupdf.fz_union_rect(block_rect, line_rect)
+        line_dict[dictkey_wmode] = line.m_internal.wmode
+        line_dict[dictkey_dir] = JM_py_from_point(line.m_internal.dir)
+        line_dict[dictkey_bbox] = JM_py_from_rect(line_rect)
+        line_list.append(line_dict)
+    block_dict[dictkey_bbox] = JM_py_from_rect(block_rect)
+    block_dict[dictkey_lines] = line_list
+
+
+def JM_make_textpage_dict(tp, page_dict, raw):
+    if g_use_extra:
+        return extra.JM_make_textpage_dict(tp.m_internal, page_dict, raw)
+    text_buffer = mupdf.fz_new_buffer(128)
+    block_list = []
+    tp_rect = mupdf.FzRect(tp.m_internal.mediabox)
+    block_n = -1
+    #log( 'JM_make_textpage_dict {=tp}')
+    for block in tp:
+        block_n += 1
+        if (not mupdf.fz_contains_rect(tp_rect, mupdf.FzRect(block.m_internal.bbox))
+                and not mupdf.fz_is_infinite_rect(tp_rect)
+                and block.m_internal.type == mupdf.FZ_STEXT_BLOCK_IMAGE
+                ):
+            continue
+        if (not mupdf.fz_is_infinite_rect(tp_rect)
+                and mupdf.fz_is_empty_rect(mupdf.fz_intersect_rect(tp_rect, mupdf.FzRect(block.m_internal.bbox)))
+                ):
+            continue
+
+        block_dict = dict()
+        block_dict[dictkey_number] = block_n
+        block_dict[dictkey_type] = block.m_internal.type
+        if block.m_internal.type == mupdf.FZ_STEXT_BLOCK_IMAGE:
+            block_dict[dictkey_bbox] = JM_py_from_rect(block.m_internal.bbox)
+            JM_make_image_block(block, block_dict)
+        else:
+            JM_make_text_block(block, block_dict, raw, text_buffer, tp_rect)
+
+        block_list.append(block_dict)
+    page_dict[dictkey_blocks] = block_list
+
+
+def JM_matrix_from_py(m):
+    a = [0, 0, 0, 0, 0, 0]
+    if isinstance(m, mupdf.FzMatrix):
+        return m
+    if isinstance(m, Matrix):
+        return mupdf.FzMatrix(m.a, m.b, m.c, m.d, m.e, m.f)
+    if not m or not PySequence_Check(m) or PySequence_Size(m) != 6:
+        return mupdf.FzMatrix()
+    for i in range(6):
+        a[i] = JM_FLOAT_ITEM(m, i)
+        if a[i] is None:
+            return mupdf.FzRect()
+    return mupdf.FzMatrix(a[0], a[1], a[2], a[3], a[4], a[5])
+
+
+def JM_mediabox(page_obj):
+    '''
+    return a PDF page's MediaBox
+    '''
+    page_mediabox = mupdf.FzRect(mupdf.FzRect.Fixed_UNIT)
+    mediabox = mupdf.pdf_to_rect(
+            mupdf.pdf_dict_get_inheritable(page_obj, PDF_NAME('MediaBox'))
+            )
+    if mupdf.fz_is_empty_rect(mediabox) or mupdf.fz_is_infinite_rect(mediabox):
+        mediabox.x0 = 0
+        mediabox.y0 = 0
+        mediabox.x1 = 612
+        mediabox.y1 = 792
+
+    page_mediabox = mupdf.FzRect(
+            mupdf.fz_min(mediabox.x0, mediabox.x1),
+            mupdf.fz_min(mediabox.y0, mediabox.y1),
+            mupdf.fz_max(mediabox.x0, mediabox.x1),
+            mupdf.fz_max(mediabox.y0, mediabox.y1),
+            )
+
+    if (page_mediabox.x1 - page_mediabox.x0 < 1
+            or page_mediabox.y1 - page_mediabox.y0 < 1
+            ):
+        page_mediabox = mupdf.FzRect(mupdf.FzRect.Fixed_UNIT)
+
+    return page_mediabox
+
+
+def JM_merge_range(
+        doc_des,
+        doc_src,
+        spage,
+        epage,
+        apage,
+        rotate,
+        links,
+        annots,
+        show_progress,
+        graft_map,
+        ):
+    '''
+    Copy a range of pages (spage, epage) from a source PDF to a specified
+    location (apage) of the target PDF.
+    If spage > epage, the sequence of source pages is reversed.
+    '''
+    if g_use_extra:
+        return extra.JM_merge_range(
+                doc_des,
+                doc_src,
+                spage,
+                epage,
+                apage,
+                rotate,
+                links,
+                annots,
+                show_progress,
+                graft_map,
+                )
+    afterpage = apage
+    counter = 0  # copied pages counter
+    total = mupdf.fz_absi(epage - spage) + 1   # total pages to copy
+
+    if spage < epage:
+        page = spage
+        while page <= epage:
+            page_merge(doc_des, doc_src, page, afterpage, rotate, links, annots, graft_map)
+            counter += 1
+            if show_progress > 0 and counter % show_progress == 0:
+                message(f"Inserted {counter} of {total} pages.")
+            page += 1
+            afterpage += 1
+    else:
+        page = spage
+        while page >= epage:
+            page_merge(doc_des, doc_src, page, afterpage, rotate, links, annots, graft_map)
+            counter += 1
+            if show_progress > 0 and counter % show_progress == 0:
+                message(f"Inserted {counter} of {total} pages.")
+            page -= 1
+            afterpage += 1
+
+
+def JM_merge_resources( page, temp_res):
+    '''
+    Merge the /Resources object created by a text pdf device into the page.
+    The device may have created multiple /ExtGState/Alp? and /Font/F? objects.
+    These need to be renamed (renumbered) to not overwrite existing page
+    objects from previous executions.
+    Returns the next available numbers n, m for objects /Alp, /F.
+    '''
+    # page objects /Resources, /Resources/ExtGState, /Resources/Font
+    resources = mupdf.pdf_dict_get(page.obj(), PDF_NAME('Resources'))
+    if not resources.m_internal:
+        resources = mupdf.pdf_dict_put_dict(page.obj(), PDF_NAME('Resources'), 5)
+    main_extg = mupdf.pdf_dict_get(resources, PDF_NAME('ExtGState'))
+    main_fonts = mupdf.pdf_dict_get(resources, PDF_NAME('Font'))
+
+    # text pdf device objects /ExtGState, /Font
+    temp_extg = mupdf.pdf_dict_get(temp_res, PDF_NAME('ExtGState'))
+    temp_fonts = mupdf.pdf_dict_get(temp_res, PDF_NAME('Font'))
+
+    max_alp = -1
+    max_fonts = -1
+
+    # Handle /Alp objects
+    if mupdf.pdf_is_dict(temp_extg):   # any created at all?
+        n = mupdf.pdf_dict_len(temp_extg)
+        if mupdf.pdf_is_dict(main_extg):   # does page have /ExtGState yet?
+            for i in range(mupdf.pdf_dict_len(main_extg)):
+                # get highest number of objects named /Alpxxx
+                alp = mupdf.pdf_to_name( mupdf.pdf_dict_get_key(main_extg, i))
+                if not alp.startswith('Alp'):
+                    continue
+                j = mupdf.fz_atoi(alp[3:])
+                if j > max_alp:
+                    max_alp = j
+        else:   # create a /ExtGState for the page
+            main_extg = mupdf.pdf_dict_put_dict(resources, PDF_NAME('ExtGState'), n)
+
+        max_alp += 1
+        for i in range(n):  # copy over renumbered /Alp objects
+            alp = mupdf.pdf_to_name( mupdf.pdf_dict_get_key( temp_extg, i))
+            j = mupdf.fz_atoi(alp[3:]) + max_alp
+            text = f'Alp{j}'
+            val = mupdf.pdf_dict_get_val( temp_extg, i)
+            mupdf.pdf_dict_puts(main_extg, text, val)
+
+    if mupdf.pdf_is_dict(main_fonts):  # has page any fonts yet?
+        for i in range(mupdf.pdf_dict_len(main_fonts)):    # get max font number
+            font = mupdf.pdf_to_name( mupdf.pdf_dict_get_key( main_fonts, i))
+            if not font.startswith("F"):
+                continue
+            j = mupdf.fz_atoi(font[1:])
+            if j > max_fonts:
+                max_fonts = j
+    else:   # create a Resources/Font for the page
+        main_fonts = mupdf.pdf_dict_put_dict(resources, PDF_NAME('Font'), 2)
+
+    max_fonts += 1
+    for i in range(mupdf.pdf_dict_len(temp_fonts)):    # copy renumbered fonts
+        font = mupdf.pdf_to_name( mupdf.pdf_dict_get_key( temp_fonts, i))
+        j = mupdf.fz_atoi(font[1:]) + max_fonts
+        text = f'F{j}'
+        val = mupdf.pdf_dict_get_val(temp_fonts, i)
+        mupdf.pdf_dict_puts(main_fonts, text, val)
+    return (max_alp, max_fonts) # next available numbers
+
+
+def JM_mupdf_warning( text):
+    '''
+    redirect MuPDF warnings
+    '''
+    JM_mupdf_warnings_store.append(text)
+    if JM_mupdf_show_warnings:
+        message(f'MuPDF warning: {text}')
+
+
+def JM_mupdf_error( text):
+    JM_mupdf_warnings_store.append(text)
+    if JM_mupdf_show_errors:
+        message(f'MuPDF error: {text}\n')
+
+
+def JM_new_bbox_device(rc, inc_layers):
+    assert isinstance(rc, list)
+    return JM_new_bbox_device_Device( rc, inc_layers)
+
+
+def JM_new_buffer_from_stext_page(page):
+    '''
+    make a buffer from an stext_page's text
+    '''
+    assert isinstance(page, mupdf.FzStextPage)
+    rect = mupdf.FzRect(page.m_internal.mediabox)
+    buf = mupdf.fz_new_buffer(256)
+    for block in page:
+        if block.m_internal.type == mupdf.FZ_STEXT_BLOCK_TEXT:
+            for line in block:
+                for ch in line:
+                    if (not JM_rects_overlap(rect, JM_char_bbox(line, ch))
+                            and not mupdf.fz_is_infinite_rect(rect)
+                            ):
+                        continue
+                    mupdf.fz_append_rune(buf, ch.m_internal.c)
+                mupdf.fz_append_byte(buf, ord('\n'))
+            mupdf.fz_append_byte(buf, ord('\n'))
+    return buf
+
+
+def JM_new_javascript(pdf, value):
+    '''
+    make new PDF action object from JavaScript source
+    Parameters are a PDF document and a Python string.
+    Returns a PDF action object.
+    '''
+    if value is None:
+        # no argument given
+        return
+    data = JM_StrAsChar(value)
+    if data is None:
+        # not convertible to char*
+        return
+
+    res = mupdf.fz_new_buffer_from_copied_data(data.encode('utf8'))
+    source = mupdf.pdf_add_stream(pdf, res, mupdf.PdfObj(), 0)
+    newaction = mupdf.pdf_add_new_dict(pdf, 4)
+    mupdf.pdf_dict_put(newaction, PDF_NAME('S'), mupdf.pdf_new_name('JavaScript'))
+    mupdf.pdf_dict_put(newaction, PDF_NAME('JS'), source)
+    return newaction
+
+
+def JM_new_output_fileptr(bio):
+    return JM_new_output_fileptr_Output( bio)
+
+
+def JM_norm_rotation(rotate):
+    '''
+    # return normalized /Rotate value:one of 0, 90, 180, 270
+    '''
+    while rotate < 0:
+        rotate += 360
+    while rotate >= 360:
+        rotate -= 360
+    if rotate % 90 != 0:
+        return 0
+    return rotate
+
+
+def JM_object_to_buffer(what, compress, ascii):
+    res = mupdf.fz_new_buffer(512)
+    out = mupdf.FzOutput(res)
+    mupdf.pdf_print_obj(out, what, compress, ascii)
+    out.fz_close_output()
+    mupdf.fz_terminate_buffer(res)
+    return res
+
+
+def JM_outline_xrefs(obj, xrefs):
+    '''
+    Return list of outline xref numbers. Recursive function. Arguments:
+    'obj' first OL item
+    'xrefs' empty Python list
+    '''
+    if not obj.m_internal:
+        return xrefs
+    thisobj = obj
+    while thisobj.m_internal:
+        newxref = mupdf.pdf_to_num( thisobj)
+        if newxref in xrefs or mupdf.pdf_dict_get( thisobj, PDF_NAME('Type')).m_internal:
+            # circular ref or top of chain: terminate
+            break
+        xrefs.append( newxref)
+        first = mupdf.pdf_dict_get( thisobj, PDF_NAME('First'))    # try go down
+        if mupdf.pdf_is_dict( first):
+            xrefs = JM_outline_xrefs( first, xrefs)
+        thisobj = mupdf.pdf_dict_get( thisobj, PDF_NAME('Next'))   # try go next
+        parent = mupdf.pdf_dict_get( thisobj, PDF_NAME('Parent'))  # get parent
+        if not mupdf.pdf_is_dict( thisobj):
+            thisobj = parent
+    return xrefs
+
+
+def JM_page_rotation(page):
+    '''
+    return a PDF page's /Rotate value: one of (0, 90, 180, 270)
+    '''
+    rotate = 0
+
+    obj = mupdf.pdf_dict_get_inheritable( page.obj(), mupdf.PDF_ENUM_NAME_Rotate)
+    rotate = mupdf.pdf_to_int(obj)
+    rotate = JM_norm_rotation(rotate)
+    return rotate
+
+
+def JM_pdf_obj_from_str(doc, src):
+    '''
+    create PDF object from given string (new in v1.14.0: MuPDF dropped it)
+    '''
+    # fixme: seems inefficient to convert to bytes instance then make another
+    # copy inside fz_new_buffer_from_copied_data(), but no other way?
+    #
+    buffer_ = mupdf.fz_new_buffer_from_copied_data(bytes(src, 'utf8'))
+    stream = mupdf.fz_open_buffer(buffer_)
+    lexbuf = mupdf.PdfLexbuf(mupdf.PDF_LEXBUF_SMALL)
+    result = mupdf.pdf_parse_stm_obj(doc, stream, lexbuf)
+    return result
+
+
+def JM_pixmap_from_display_list(
+        list_,
+        ctm,
+        cs,
+        alpha,
+        clip,
+        seps,
+        ):
+    '''
+    Version of fz_new_pixmap_from_display_list (util.c) to also support
+    rendering of only the 'clip' part of the displaylist rectangle
+    '''
+    assert isinstance(list_, mupdf.FzDisplayList)
+    if seps is None:
+        seps = mupdf.FzSeparations()
+    assert seps is None or isinstance(seps, mupdf.FzSeparations), f'{type(seps)=}: {seps}'
+
+    rect = mupdf.fz_bound_display_list(list_)
+    matrix = JM_matrix_from_py(ctm)
+    rclip = JM_rect_from_py(clip)
+    rect = mupdf.fz_intersect_rect(rect, rclip)    # no-op if clip is not given
+
+    rect = mupdf.fz_transform_rect(rect, matrix)
+    irect = mupdf.fz_round_rect(rect)
+
+    assert isinstance( cs, mupdf.FzColorspace)
+
+    pix = mupdf.fz_new_pixmap_with_bbox(cs, irect, seps, alpha)
+    if alpha:
+        mupdf.fz_clear_pixmap(pix)
+    else:
+        mupdf.fz_clear_pixmap_with_value(pix, 0xFF)
+
+    if not mupdf.fz_is_infinite_rect(rclip):
+        dev = mupdf.fz_new_draw_device_with_bbox(matrix, pix, irect)
+        mupdf.fz_run_display_list(list_, dev, mupdf.FzMatrix(), rclip, mupdf.FzCookie())
+    else:
+        dev = mupdf.fz_new_draw_device(matrix, pix)
+        mupdf.fz_run_display_list(list_, dev, mupdf.FzMatrix(), mupdf.FzRect(mupdf.FzRect.Fixed_INFINITE), mupdf.FzCookie())
+
+    mupdf.fz_close_device(dev)
+    # Use special raw Pixmap constructor so we don't set alpha to true.
+    return Pixmap( 'raw', pix)
+
+
+def JM_point_from_py(p):
+    '''
+    PySequence to fz_point. Default: (FZ_MIN_INF_RECT, FZ_MIN_INF_RECT)
+    '''
+    if isinstance(p, mupdf.FzPoint):
+        return p
+    if isinstance(p, Point):
+        return mupdf.FzPoint(p.x, p.y)
+    if g_use_extra:
+        return extra.JM_point_from_py( p)
+    
+    p0 = mupdf.FzPoint(0, 0)
+    x = JM_FLOAT_ITEM(p, 0)
+    y = JM_FLOAT_ITEM(p, 1)
+    if x is None or y is None:
+        return p0
+    x = max( x, FZ_MIN_INF_RECT)
+    y = max( y, FZ_MIN_INF_RECT)
+    x = min( x, FZ_MAX_INF_RECT)
+    y = min( y, FZ_MAX_INF_RECT)
+    return mupdf.FzPoint(x, y)
+
+
+def JM_print_stext_page_as_text(res, page):
+    '''
+    Plain text output. An identical copy of fz_print_stext_page_as_text,
+    but lines within a block are concatenated by space instead a new-line
+    character (which else leads to 2 new-lines).
+    '''
+    if 1 and g_use_extra:
+        return extra.JM_print_stext_page_as_text(res, page)
+    
+    assert isinstance(res, mupdf.FzBuffer)
+    assert isinstance(page, mupdf.FzStextPage)
+    rect = mupdf.FzRect(page.m_internal.mediabox)
+    last_char = 0
+
+    n_blocks = 0
+    n_lines = 0
+    n_chars = 0
+    for n_blocks2, block in enumerate( page):
+        if block.m_internal.type == mupdf.FZ_STEXT_BLOCK_TEXT:
+            for n_lines2, line in enumerate( block):
+                for n_chars2, ch in enumerate( line):
+                    pass
+                n_chars += n_chars2
+            n_lines += n_lines2
+        n_blocks += n_blocks2
+    
+    for block in page:
+        if block.m_internal.type == mupdf.FZ_STEXT_BLOCK_TEXT:
+            for line in block:
+                last_char = 0
+                for ch in line:
+                    chbbox = JM_char_bbox(line, ch)
+                    if (mupdf.fz_is_infinite_rect(rect)
+                            or JM_rects_overlap(rect, chbbox)
+                            ):
+                        #raw += chr(ch.m_internal.c)
+                        last_char = ch.m_internal.c
+                        #log( '{=last_char!r utf!r}')
+                        JM_append_rune(res, last_char)
+                if last_char != 10 and last_char > 0:
+                    mupdf.fz_append_string(res, "\n")
+
+
+def JM_put_script(annot_obj, key1, key2, value):
+    '''
+    Create a JavaScript PDF action.
+    Usable for all object types which support PDF actions, even if the
+    argument name suggests annotations. Up to 2 key values can be specified, so
+    JavaScript actions can be stored for '/A' and '/AA/?' keys.
+    '''
+    key1_obj = mupdf.pdf_dict_get(annot_obj, key1)
+    pdf = mupdf.pdf_get_bound_document(annot_obj)  # owning PDF
+
+    # if no new script given, just delete corresponding key
+    if not value:
+        if key2 is None or not key2.m_internal:
+            mupdf.pdf_dict_del(annot_obj, key1)
+        elif key1_obj.m_internal:
+            mupdf.pdf_dict_del(key1_obj, key2)
+        return
+
+    # read any existing script as a PyUnicode string
+    if not key2.m_internal or not key1_obj.m_internal:
+        script = JM_get_script(key1_obj)
+    else:
+        script = JM_get_script(mupdf.pdf_dict_get(key1_obj, key2))
+
+    # replace old script, if different from new one
+    if value != script:
+        newaction = JM_new_javascript(pdf, value)
+        if not key2.m_internal:
+            mupdf.pdf_dict_put(annot_obj, key1, newaction)
+        else:
+            mupdf.pdf_dict_putl(annot_obj, newaction, key1, key2)
+
+
+def JM_py_from_irect(r):
+    return r.x0, r.y0, r.x1, r.y1
+
+
+def JM_py_from_matrix(m):
+    return m.a, m.b, m.c, m.d, m.e, m.f
+
+
+def JM_py_from_point(p):
+    return p.x, p.y
+
+
+def JM_py_from_quad(q):
+    '''
+    PySequence from fz_quad.
+    '''
+    return (
+            (q.ul.x, q.ul.y),
+            (q.ur.x, q.ur.y),
+            (q.ll.x, q.ll.y),
+            (q.lr.x, q.lr.y),
+            )
+
+
+def JM_py_from_rect(r):
+    return r.x0, r.y0, r.x1, r.y1
+
+
+def JM_quad_from_py(r):
+    if isinstance(r, mupdf.FzQuad):
+        return r
+    # cover all cases of 4-float-sequences
+    if hasattr(r, "__getitem__") and len(r) == 4 and hasattr(r[0], "__float__"):
+        r = mupdf.FzRect(*tuple(r))
+    if isinstance( r, mupdf.FzRect):
+        return mupdf.fz_quad_from_rect( r)
+    if isinstance( r, Quad):
+        return mupdf.fz_make_quad(
+                r.ul.x, r.ul.y,
+                r.ur.x, r.ur.y,
+                r.ll.x, r.ll.y,
+                r.lr.x, r.lr.y,
+                )
+    q = mupdf.fz_make_quad(0, 0, 0, 0, 0, 0, 0, 0)
+    p = [0,0,0,0]
+    if not r or not isinstance(r, (tuple, list)) or len(r) != 4:
+        return q
+
+    if JM_FLOAT_ITEM(r, 0) is None:
+        return mupdf.fz_quad_from_rect(JM_rect_from_py(r))
+
+    for i in range(4):
+        if i >= len(r):
+            return q    # invalid: cancel the rest
+        obj = r[i]  # next point item
+        if not PySequence_Check(obj) or PySequence_Size(obj) != 2:
+            return q    # invalid: cancel the rest
+
+        p[i].x = JM_FLOAT_ITEM(obj, 0)
+        p[i].y = JM_FLOAT_ITEM(obj, 1)
+        if p[i].x is None or p[i].y is None:
+            return q
+        p[i].x = max( p[i].x, FZ_MIN_INF_RECT)
+        p[i].y = max( p[i].y, FZ_MIN_INF_RECT)
+        p[i].x = min( p[i].x, FZ_MAX_INF_RECT)
+        p[i].y = min( p[i].y, FZ_MAX_INF_RECT)
+    q.ul = p[0]
+    q.ur = p[1]
+    q.ll = p[2]
+    q.lr = p[3]
+    return q
+
+
+def JM_read_contents(pageref):
+    '''
+    Read and concatenate a PDF page's /Contents object(s) in a buffer
+    '''
+    assert isinstance(pageref, mupdf.PdfObj), f'{type(pageref)}'
+    contents = mupdf.pdf_dict_get(pageref, mupdf.PDF_ENUM_NAME_Contents)
+    if mupdf.pdf_is_array(contents):
+        res = mupdf.FzBuffer(1024)
+        for i in range(mupdf.pdf_array_len(contents)):
+            if i > 0:
+                mupdf.fz_append_byte(res, 32)
+            obj = mupdf.pdf_array_get(contents, i)
+            if mupdf.pdf_is_stream(obj):
+                nres = mupdf.pdf_load_stream(obj)
+                mupdf.fz_append_buffer(res, nres)
+    elif contents.m_internal:
+        res = mupdf.pdf_load_stream(contents)
+    else:
+        res = mupdf.FzBuffer(0)
+    return res
+
+
+def JM_rect_from_py(r):
+    if isinstance(r, mupdf.FzRect):
+        return r
+    if isinstance(r, mupdf.FzIrect):
+        return mupdf.FzRect(r)
+    if isinstance(r, Rect):
+        return mupdf.fz_make_rect(r.x0, r.y0, r.x1, r.y1)
+    if isinstance(r, IRect):
+        return mupdf.fz_make_rect(r.x0, r.y0, r.x1, r.y1)
+    if not r or not PySequence_Check(r) or PySequence_Size(r) != 4:
+        return mupdf.FzRect(mupdf.FzRect.Fixed_INFINITE)
+    f = [0, 0, 0, 0]
+    for i in range(4):
+        f[i] = JM_FLOAT_ITEM(r, i)
+        if f[i] is None:
+            return mupdf.FzRect(mupdf.FzRect.Fixed_INFINITE)
+        if f[i] < FZ_MIN_INF_RECT:
+            f[i] = FZ_MIN_INF_RECT
+        if f[i] > FZ_MAX_INF_RECT:
+            f[i] = FZ_MAX_INF_RECT
+    return mupdf.fz_make_rect(f[0], f[1], f[2], f[3])
+
+
+def JM_rects_overlap(a, b):
+    if (0
+            or a.x0 >= b.x1
+            or a.y0 >= b.y1
+            or a.x1 <= b.x0
+            or a.y1 <= b.y0
+            ):
+        return 0
+    return 1
+
+
+def JM_refresh_links( page):
+    '''
+    refreshes the link and annotation tables of a page
+    '''
+    if page is None or not page.m_internal:
+        return
+    obj = mupdf.pdf_dict_get( page.obj(), PDF_NAME('Annots'))
+    if obj.m_internal:
+        pdf = page.doc()
+        number = mupdf.pdf_lookup_page_number( pdf, page.obj())
+        page_mediabox = mupdf.FzRect()
+        page_ctm = mupdf.FzMatrix()
+        mupdf.pdf_page_transform( page, page_mediabox, page_ctm)
+        link = mupdf.pdf_load_link_annots( pdf, page, obj, number, page_ctm)
+        page.m_internal.links = mupdf.ll_fz_keep_link( link.m_internal)
+
+
+def JM_rotate_page_matrix(page):
+    '''
+    calculate page rotation matrices
+    '''
+    if not page.m_internal:
+        return mupdf.FzMatrix()  # no valid pdf page given
+    rotation = JM_page_rotation(page)
+    #log( '{rotation=}')
+    if rotation == 0:
+        return mupdf.FzMatrix()  # no rotation
+    cb_size = JM_cropbox_size(page.obj())
+    w = cb_size.x
+    h = cb_size.y
+    #log( '{=h w}')
+    if rotation == 90:
+        m = mupdf.fz_make_matrix(0, 1, -1, 0, h, 0)
+    elif rotation == 180:
+        m = mupdf.fz_make_matrix(-1, 0, 0, -1, w, h)
+    else:
+        m = mupdf.fz_make_matrix(0, -1, 1, 0, 0, w)
+    #log( 'returning {m=}')
+    return m
+
+
+def JM_search_stext_page(page, needle):
+    if g_use_extra:
+        return extra.JM_search_stext_page(page.m_internal, needle)
+    
+    rect = mupdf.FzRect(page.m_internal.mediabox)
+    if not needle:
+        return
+    quads = []
+    class Hits:
+        def __str__(self):
+            return f'Hits(len={self.len} quads={self.quads} hfuzz={self.hfuzz} vfuzz={self.vfuzz}'
+    hits = Hits()
+    hits.len = 0
+    hits.quads = quads
+    hits.hfuzz = 0.2    # merge kerns but not large gaps
+    hits.vfuzz = 0.1
+
+    buffer_ = JM_new_buffer_from_stext_page(page)
+    haystack_string = mupdf.fz_string_from_buffer(buffer_)
+    haystack = 0
+    begin, end = find_string(haystack_string[haystack:], needle)
+    if begin is None:
+        #goto no_more_matches;
+        return quads
+
+    begin += haystack
+    end += haystack
+    inside = 0
+    i = 0
+    for block in page:
+        if block.m_internal.type != mupdf.FZ_STEXT_BLOCK_TEXT:
+            continue
+        for line in block:
+            for ch in line:
+                i += 1
+                if not mupdf.fz_is_infinite_rect(rect):
+                    r = JM_char_bbox(line, ch)
+                    if not JM_rects_overlap(rect, r):
+                        #goto next_char;
+                        continue
+                while 1:
+                    #try_new_match:
+                    if not inside:
+                        if haystack >= begin:
+                            inside = 1
+                    if inside:
+                        if haystack < end:
+                            on_highlight_char(hits, line, ch)
+                            break
+                        else:
+                            inside = 0
+                            begin, end = find_string(haystack_string[haystack:], needle)
+                            if begin is None:
+                                #goto no_more_matches;
+                                return quads
+                            else:
+                                #goto try_new_match;
+                                begin += haystack
+                                end += haystack
+                                continue
+                    break
+                haystack += 1
+                #next_char:;
+            assert haystack_string[haystack] == '\n', \
+                    f'{haystack=} {haystack_string[haystack]=}'
+            haystack += 1
+        assert haystack_string[haystack] == '\n', \
+                f'{haystack=} {haystack_string[haystack]=}'
+        haystack += 1
+    #no_more_matches:;
+    return quads
+
+
+def JM_scan_resources(pdf, rsrc, liste, what, stream_xref, tracer):
+    '''
+    Step through /Resources, looking up image, xobject or font information
+    '''
+    if mupdf.pdf_mark_obj(rsrc):
+        mupdf.fz_warn('Circular dependencies! Consider page cleaning.')
+        return  # Circular dependencies!
+    try:
+        xobj = mupdf.pdf_dict_get(rsrc, mupdf.PDF_ENUM_NAME_XObject)
+
+        if what == 1:   # lookup fonts
+            font = mupdf.pdf_dict_get(rsrc, mupdf.PDF_ENUM_NAME_Font)
+            JM_gather_fonts(pdf, font, liste, stream_xref)
+        elif what == 2: # look up images
+            JM_gather_images(pdf, xobj, liste, stream_xref)
+        elif what == 3: # look up form xobjects
+            JM_gather_forms(pdf, xobj, liste, stream_xref)
+        else:   # should never happen
+            return
+
+        # check if we need to recurse into Form XObjects
+        n = mupdf.pdf_dict_len(xobj)
+        for i in range(n):
+            obj = mupdf.pdf_dict_get_val(xobj, i)
+            if mupdf.pdf_is_stream(obj):
+                sxref = mupdf.pdf_to_num(obj)
+            else:
+                sxref = 0
+            subrsrc = mupdf.pdf_dict_get(obj, mupdf.PDF_ENUM_NAME_Resources)
+            if subrsrc.m_internal:
+                sxref_t = sxref
+                if sxref_t not in tracer:
+                    tracer.append(sxref_t)
+                    JM_scan_resources( pdf, subrsrc, liste, what, sxref, tracer)
+                else:
+                    mupdf.fz_warn('Circular dependencies! Consider page cleaning.')
+                    return
+    finally:
+        mupdf.pdf_unmark_obj(rsrc)
+
+
+def JM_set_choice_options(annot, liste):
+    '''
+    set ListBox / ComboBox values
+    '''
+    if not liste:
+        return
+    assert isinstance( liste, (tuple, list))
+    n = len( liste)
+    if n == 0:
+        return
+    annot_obj = mupdf.pdf_annot_obj( annot)
+    pdf = mupdf.pdf_get_bound_document( annot_obj)
+    optarr = mupdf.pdf_new_array( pdf, n)
+    for i in range(n):
+        val = liste[i]
+        opt = val
+        if isinstance(opt, str):
+            mupdf.pdf_array_push_text_string( optarr, opt)
+        else:
+            assert isinstance( val, (tuple, list)) and len( val) == 2, 'bad choice field list'
+            opt1, opt2 = val
+            assert opt1 and opt2, 'bad choice field list'
+            optarrsub = mupdf.pdf_array_push_array( optarr, 2)
+            mupdf.pdf_array_push_text_string( optarrsub, opt1)
+            mupdf.pdf_array_push_text_string( optarrsub, opt2)
+    mupdf.pdf_dict_put( annot_obj, PDF_NAME('Opt'), optarr)
+
+
+def JM_set_field_type(doc, obj, type):
+    '''
+    Set the field type
+    '''
+    setbits = 0
+    clearbits = 0
+    typename = None
+    if type == mupdf.PDF_WIDGET_TYPE_BUTTON:
+        typename = PDF_NAME('Btn')
+        setbits = mupdf.PDF_BTN_FIELD_IS_PUSHBUTTON
+    elif type == mupdf.PDF_WIDGET_TYPE_RADIOBUTTON:
+        typename = PDF_NAME('Btn')
+        clearbits = mupdf.PDF_BTN_FIELD_IS_PUSHBUTTON
+        setbits = mupdf.PDF_BTN_FIELD_IS_RADIO
+    elif type == mupdf.PDF_WIDGET_TYPE_CHECKBOX:
+        typename = PDF_NAME('Btn')
+        clearbits = (mupdf.PDF_BTN_FIELD_IS_PUSHBUTTON | mupdf.PDF_BTN_FIELD_IS_RADIO)
+    elif type == mupdf.PDF_WIDGET_TYPE_TEXT:
+        typename = PDF_NAME('Tx')
+    elif type == mupdf.PDF_WIDGET_TYPE_LISTBOX:
+        typename = PDF_NAME('Ch')
+        clearbits = mupdf.PDF_CH_FIELD_IS_COMBO
+    elif type == mupdf.PDF_WIDGET_TYPE_COMBOBOX:
+        typename = PDF_NAME('Ch')
+        setbits = mupdf.PDF_CH_FIELD_IS_COMBO
+    elif type == mupdf.PDF_WIDGET_TYPE_SIGNATURE:
+        typename = PDF_NAME('Sig')
+
+    if typename is not None and typename.m_internal:
+        mupdf.pdf_dict_put(obj, PDF_NAME('FT'), typename)
+
+    if setbits != 0 or clearbits != 0:
+        bits = mupdf.pdf_dict_get_int(obj, PDF_NAME('Ff'))
+        bits &= ~clearbits
+        bits |= setbits
+        mupdf.pdf_dict_put_int(obj, PDF_NAME('Ff'), bits)
+
+
+def JM_set_object_value(obj, key, value):
+    '''
+    Set a PDF dict key to some value
+    '''
+    eyecatcher = "fitz: replace me!"
+    pdf = mupdf.pdf_get_bound_document(obj)
+    # split PDF key at path seps and take last key part
+    list_ = key.split('/')
+    len_ = len(list_)
+    i = len_ - 1
+    skey = list_[i]
+
+    del list_[i]    # del the last sub-key
+    len_ = len(list_)   # remaining length
+    testkey = mupdf.pdf_dict_getp(obj, key)    # check if key already exists
+    if not testkey.m_internal:
+        #No, it will be created here. But we cannot allow this happening if
+        #indirect objects are referenced. So we check all higher level
+        #sub-paths for indirect references.
+        while len_ > 0:
+            t = '/'.join(list_) # next high level
+            if mupdf.pdf_is_indirect(mupdf.pdf_dict_getp(obj, JM_StrAsChar(t))):
+                raise Exception("path to '%s' has indirects", JM_StrAsChar(skey))
+            del list_[len_ - 1]   # del last sub-key
+            len_ = len(list_)   # remaining length
+    # Insert our eyecatcher. Will create all sub-paths in the chain, or
+    # respectively remove old value of key-path.
+    mupdf.pdf_dict_putp(obj, key, mupdf.pdf_new_text_string(eyecatcher))
+    testkey = mupdf.pdf_dict_getp(obj, key)
+    if not mupdf.pdf_is_string(testkey):
+        raise Exception("cannot insert value for '%s'", key)
+    temp = mupdf.pdf_to_text_string(testkey)
+    if temp != eyecatcher:
+        raise Exception("cannot insert value for '%s'", key)
+    # read the result as a string
+    res = JM_object_to_buffer(obj, 1, 0)
+    objstr = JM_EscapeStrFromBuffer(res)
+
+    # replace 'eyecatcher' by desired 'value'
+    nullval = "/%s(%s)" % ( skey, eyecatcher)
+    newval = "/%s %s" % (skey, value)
+    newstr = objstr.replace(nullval, newval, 1)
+
+    # make PDF object from resulting string
+    new_obj = JM_pdf_obj_from_str(pdf, newstr)
+    return new_obj
+
+
+def JM_set_ocg_arrays(conf, basestate, on, off, rbgroups, locked):
+    if basestate:
+        mupdf.pdf_dict_put_name( conf, PDF_NAME('BaseState'), basestate)
+
+    if on is not None:
+        mupdf.pdf_dict_del( conf, PDF_NAME('ON'))
+        if on:
+            arr = mupdf.pdf_dict_put_array( conf, PDF_NAME('ON'), 1)
+            JM_set_ocg_arrays_imp( arr, on)
+    if off is not None:
+        mupdf.pdf_dict_del( conf, PDF_NAME('OFF'))
+        if off:
+            arr = mupdf.pdf_dict_put_array( conf, PDF_NAME('OFF'), 1)
+            JM_set_ocg_arrays_imp( arr, off)
+    if locked is not None:
+        mupdf.pdf_dict_del( conf, PDF_NAME('Locked'))
+        if locked:
+            arr = mupdf.pdf_dict_put_array( conf, PDF_NAME('Locked'), 1)
+            JM_set_ocg_arrays_imp( arr, locked)
+    if rbgroups is not None:
+        mupdf.pdf_dict_del( conf, PDF_NAME('RBGroups'))
+        if rbgroups:
+            arr = mupdf.pdf_dict_put_array( conf, PDF_NAME('RBGroups'), 1)
+            n =len(rbgroups)
+            for i in range(n):
+                item0 = rbgroups[i]
+                obj = mupdf.pdf_array_push_array( arr, 1)
+                JM_set_ocg_arrays_imp( obj, item0)
+
+
+def JM_set_ocg_arrays_imp(arr, list_):
+    '''
+    Set OCG arrays from dict of Python lists
+    Works with dict like {"basestate":name, "on":list, "off":list, "rbg":list}
+    '''
+    pdf = mupdf.pdf_get_bound_document(arr)
+    for xref in list_:
+        obj = mupdf.pdf_new_indirect(pdf, xref, 0)
+        mupdf.pdf_array_push(arr, obj)
+
+
+def JM_set_resource_property(ref, name, xref):
+    '''
+    Insert an item into Resources/Properties (used for Marked Content)
+    Arguments:
+    (1) e.g. page object, Form XObject
+    (2) marked content name
+    (3) xref of the referenced object (insert as indirect reference)
+    '''
+    pdf = mupdf.pdf_get_bound_document(ref)
+    ind = mupdf.pdf_new_indirect(pdf, xref, 0)
+    if not ind.m_internal:
+        RAISEPY(MSG_BAD_XREF, PyExc_ValueError)
+    resources = mupdf.pdf_dict_get(ref, PDF_NAME('Resources'))
+    if not resources.m_internal:
+        resources = mupdf.pdf_dict_put_dict(ref, PDF_NAME('Resources'), 1)
+    properties = mupdf.pdf_dict_get(resources, PDF_NAME('Properties'))
+    if not properties.m_internal:
+        properties = mupdf.pdf_dict_put_dict(resources, PDF_NAME('Properties'), 1)
+    mupdf.pdf_dict_put(properties, mupdf.pdf_new_name(name), ind)
+
+
+def JM_set_widget_properties(annot, Widget):
+    '''
+    Update the PDF form field with the properties from a Python Widget object.
+    Called by "Page.add_widget" and "Annot.update_widget".
+    '''
+    if isinstance( annot, Annot):
+        annot = annot.this
+    assert isinstance( annot, mupdf.PdfAnnot), f'{type(annot)=} {type=}'
+    page = _pdf_annot_page(annot)
+    assert page.m_internal, 'Annot is not bound to a page'
+    annot_obj = mupdf.pdf_annot_obj(annot)
+    pdf = page.doc()
+    def GETATTR(name):
+        return getattr(Widget, name, None)
+
+    value = GETATTR("field_type")
+    field_type = value
+
+    # rectangle --------------------------------------------------------------
+    value = GETATTR("rect")
+    rect = JM_rect_from_py(value)
+    rot_mat = JM_rotate_page_matrix(page)
+    rect = mupdf.fz_transform_rect(rect, rot_mat)
+    mupdf.pdf_set_annot_rect(annot, rect)
+
+    # fill color -------------------------------------------------------------
+    value = GETATTR("fill_color")
+    if value and PySequence_Check(value):
+        n = len(value)
+        fill_col = mupdf.pdf_new_array(pdf, n)
+        col = 0
+        for i in range(n):
+            col = value[i]
+            mupdf.pdf_array_push_real(fill_col, col)
+        mupdf.pdf_field_set_fill_color(annot_obj, fill_col)
+
+    # dashes -----------------------------------------------------------------
+    value = GETATTR("border_dashes")
+    if value and PySequence_Check(value):
+        n = len(value)
+        dashes = mupdf.pdf_new_array(pdf, n)
+        for i in range(n):
+            mupdf.pdf_array_push_int(dashes, value[i])
+        mupdf.pdf_dict_putl(annot_obj, dashes, PDF_NAME('BS'), PDF_NAME('D'))
+
+    # border color -----------------------------------------------------------
+    value = GETATTR("border_color")
+    if value and PySequence_Check(value):
+        n = len(value)
+        border_col = mupdf.pdf_new_array(pdf, n)
+        col = 0
+        for i in range(n):
+            col = value[i]
+            mupdf.pdf_array_push_real(border_col, col)
+        mupdf.pdf_dict_putl(annot_obj, border_col, PDF_NAME('MK'), PDF_NAME('BC'))
+
+    # entry ignored - may be used later
+    #
+    #int text_format = (int) PyInt_AsLong(GETATTR("text_format"));
+    #
+
+    # field label -----------------------------------------------------------
+    value = GETATTR("field_label")
+    if value is not None:
+        label = JM_StrAsChar(value)
+        mupdf.pdf_dict_put_text_string(annot_obj, PDF_NAME('TU'), label)
+
+    # field name -------------------------------------------------------------
+    value = GETATTR("field_name")
+    if value is not None:
+        name = JM_StrAsChar(value)
+        old_name = mupdf.pdf_load_field_name(annot_obj)
+        if name != old_name:
+            mupdf.pdf_dict_put_text_string(annot_obj, PDF_NAME('T'), name)
+
+    # max text len -----------------------------------------------------------
+    if field_type == mupdf.PDF_WIDGET_TYPE_TEXT:
+        value = GETATTR("text_maxlen")
+        text_maxlen = value
+        if text_maxlen:
+            mupdf.pdf_dict_put_int(annot_obj, PDF_NAME('MaxLen'), text_maxlen)
+    value = GETATTR("field_display")
+    d = value
+    mupdf.pdf_field_set_display(annot_obj, d)
+
+    # choice values ----------------------------------------------------------
+    if field_type in (mupdf.PDF_WIDGET_TYPE_LISTBOX, mupdf.PDF_WIDGET_TYPE_COMBOBOX):
+        value = GETATTR("choice_values")
+        JM_set_choice_options(annot, value)
+
+    # border style -----------------------------------------------------------
+    value = GETATTR("border_style")
+    val = JM_get_border_style(value)
+    mupdf.pdf_dict_putl(annot_obj, val, PDF_NAME('BS'), PDF_NAME('S'))
+
+    # border width -----------------------------------------------------------
+    value = GETATTR("border_width")
+    border_width = value
+    mupdf.pdf_dict_putl(
+            annot_obj,
+            mupdf.pdf_new_real(border_width),
+            PDF_NAME('BS'),
+            PDF_NAME('W'),
+            )
+
+    # /DA string -------------------------------------------------------------
+    value = GETATTR("_text_da")
+    da = JM_StrAsChar(value)
+    mupdf.pdf_dict_put_text_string(annot_obj, PDF_NAME('DA'), da)
+    mupdf.pdf_dict_del(annot_obj, PDF_NAME('DS'))  # not supported by MuPDF
+    mupdf.pdf_dict_del(annot_obj, PDF_NAME('RC'))  # not supported by MuPDF
+
+    # field flags ------------------------------------------------------------
+    field_flags = GETATTR("field_flags")
+    if field_flags is not None:
+        if field_type == mupdf.PDF_WIDGET_TYPE_COMBOBOX:
+            field_flags |= mupdf.PDF_CH_FIELD_IS_COMBO
+        elif field_type == mupdf.PDF_WIDGET_TYPE_RADIOBUTTON:
+            field_flags |= mupdf.PDF_BTN_FIELD_IS_RADIO
+        elif field_type == mupdf.PDF_WIDGET_TYPE_BUTTON:
+            field_flags |= mupdf.PDF_BTN_FIELD_IS_PUSHBUTTON
+        mupdf.pdf_dict_put_int( annot_obj, PDF_NAME('Ff'), field_flags)
+
+    # button caption ---------------------------------------------------------
+    value = GETATTR("button_caption")
+    ca = JM_StrAsChar(value)
+    if ca:
+        mupdf.pdf_field_set_button_caption(annot_obj, ca)
+
+    # script (/A) -------------------------------------------------------
+    value = GETATTR("script")
+    JM_put_script(annot_obj, PDF_NAME('A'), mupdf.PdfObj(), value)
+
+    # script (/AA/K) -------------------------------------------------------
+    value = GETATTR("script_stroke")
+    JM_put_script(annot_obj, PDF_NAME('AA'), PDF_NAME('K'), value)
+
+    # script (/AA/F) -------------------------------------------------------
+    value = GETATTR("script_format")
+    JM_put_script(annot_obj, PDF_NAME('AA'), PDF_NAME('F'), value)
+
+    # script (/AA/V) -------------------------------------------------------
+    value = GETATTR("script_change")
+    JM_put_script(annot_obj, PDF_NAME('AA'), PDF_NAME('V'), value)
+
+    # script (/AA/C) -------------------------------------------------------
+    value = GETATTR("script_calc")
+    JM_put_script(annot_obj, PDF_NAME('AA'), PDF_NAME('C'), value)
+
+    # script (/AA/Bl) -------------------------------------------------------
+    value = GETATTR("script_blur")
+    JM_put_script(annot_obj, PDF_NAME('AA'), mupdf.pdf_new_name('Bl'), value)
+
+    # script (/AA/Fo) codespell:ignore --------------------------------------
+    value = GETATTR("script_focus")
+    JM_put_script(annot_obj, PDF_NAME('AA'), mupdf.pdf_new_name('Fo'), value)
+
+    # field value ------------------------------------------------------------
+    value = GETATTR("field_value")  # field value
+    text = JM_StrAsChar(value)  # convert to text (may fail!)
+    if field_type == mupdf.PDF_WIDGET_TYPE_RADIOBUTTON:
+        if not value:
+            mupdf.pdf_set_field_value(pdf, annot_obj, "Off", 1)
+            mupdf.pdf_dict_put_name(annot_obj, PDF_NAME('AS'), "Off")
+        else:
+            # TODO check if another button in the group is ON and if so set it Off
+            onstate = mupdf.pdf_button_field_on_state(annot_obj)
+            if onstate.m_internal:
+                on = mupdf.pdf_to_name(onstate)
+                mupdf.pdf_set_field_value(pdf, annot_obj, on, 1)
+                mupdf.pdf_dict_put_name(annot_obj, PDF_NAME('AS'), on)
+            elif text:
+                mupdf.pdf_dict_put_name(annot_obj, PDF_NAME('AS'), text)
+    elif field_type == mupdf.PDF_WIDGET_TYPE_CHECKBOX:
+        onstate = mupdf.pdf_button_field_on_state(annot_obj)
+        on = onstate.pdf_to_name()
+        if value in (True, on) or text == 'Yes':
+            mupdf.pdf_set_field_value(pdf, annot_obj, on, 1)
+            mupdf.pdf_dict_put_name(annot_obj, PDF_NAME('AS'), on)
+            mupdf.pdf_dict_put_name(annot_obj, PDF_NAME('V'), on)
+        else:
+            mupdf.pdf_dict_put_name( annot_obj, PDF_NAME('AS'), 'Off')
+            mupdf.pdf_dict_put_name( annot_obj, PDF_NAME('V'), 'Off')
+    else:
+        if text:
+            mupdf.pdf_set_field_value(pdf, annot_obj, text, 1)
+            if field_type in (mupdf.PDF_WIDGET_TYPE_COMBOBOX, mupdf.PDF_WIDGET_TYPE_LISTBOX):
+                mupdf.pdf_dict_del(annot_obj, PDF_NAME('I'))
+    mupdf.pdf_dirty_annot(annot)
+    mupdf.pdf_set_annot_hot(annot, 1)
+    mupdf.pdf_set_annot_active(annot, 1)
+    mupdf.pdf_update_annot(annot)
+
+
+def JM_show_string_cs(
+        text,
+        user_font,
+        trm,
+        s,
+        wmode,
+        bidi_level,
+        markup_dir,
+        language,
+        ):
+    i = 0
+    while i < len(s):
+        l, ucs = mupdf.fz_chartorune(s[i:])
+        i += l
+        gid = mupdf.fz_encode_character_sc(user_font, ucs)
+        if gid == 0:
+            gid, font = mupdf.fz_encode_character_with_fallback(user_font, ucs, 0, language)
+        else:
+            font = user_font
+        mupdf.fz_show_glyph(text, font, trm, gid, ucs, wmode, bidi_level, markup_dir, language)
+        adv = mupdf.fz_advance_glyph(font, gid, wmode)
+        if wmode == 0:
+            trm = mupdf.fz_pre_translate(trm, adv, 0)
+        else:
+            trm = mupdf.fz_pre_translate(trm, 0, -adv)
+    return trm
+
+
+def JM_UnicodeFromBuffer(buff):
+    buff_bytes = mupdf.fz_buffer_extract_copy(buff)
+    val = buff_bytes.decode(errors='replace')
+    z = val.find(chr(0))
+    if z >= 0:
+        val = val[:z]
+    return val
+
+
+def message_warning(text):
+    '''
+    Generate a warning.
+    '''
+    message(f'warning: {text}')
+
+
+def JM_update_stream(doc, obj, buffer_, compress):
+    '''
+    update a stream object
+    compress stream when beneficial
+    '''
+    if compress:
+        length, _ = mupdf.fz_buffer_storage(buffer_)
+        if length > 30:   # ignore small stuff
+            buffer_compressed = JM_compress_buffer(buffer_)
+            assert isinstance(buffer_compressed, mupdf.FzBuffer)
+            if buffer_compressed.m_internal:
+                length_compressed, _ = mupdf.fz_buffer_storage(buffer_compressed)
+                if length_compressed < length:  # was it worth the effort?
+                    mupdf.pdf_dict_put(
+                            obj,
+                            mupdf.PDF_ENUM_NAME_Filter,
+                            mupdf.PDF_ENUM_NAME_FlateDecode,
+                            )
+                    mupdf.pdf_update_stream(doc, obj, buffer_compressed, 1)
+                    return
+    
+    mupdf.pdf_update_stream(doc, obj, buffer_, 0)
+
+
+def JM_xobject_from_page(pdfout, fsrcpage, xref, gmap):
+    '''
+    Make an XObject from a PDF page
+    For a positive xref assume that its object can be used instead
+    '''
+    assert isinstance(gmap, mupdf.PdfGraftMap), f'{type(gmap)=}'
+    if xref > 0:
+        xobj1 = mupdf.pdf_new_indirect(pdfout, xref, 0)
+    else:
+        srcpage = _as_pdf_page(fsrcpage.this)
+        spageref = srcpage.obj()
+        mediabox = mupdf.pdf_to_rect(mupdf.pdf_dict_get_inheritable(spageref, PDF_NAME('MediaBox')))
+        # Deep-copy resources object of source page
+        o = mupdf.pdf_dict_get_inheritable(spageref, PDF_NAME('Resources'))
+        if gmap.m_internal:
+            # use graftmap when possible
+            resources = mupdf.pdf_graft_mapped_object(gmap, o)
+        else:
+            resources = mupdf.pdf_graft_object(pdfout, o)
+
+        # get spgage contents source
+        res = JM_read_contents(spageref)
+
+        #-------------------------------------------------------------
+        # create XObject representing the source page
+        #-------------------------------------------------------------
+        xobj1 = mupdf.pdf_new_xobject(pdfout, mediabox, mupdf.FzMatrix(), mupdf.PdfObj(0), res)
+        # store spage contents
+        JM_update_stream(pdfout, xobj1, res, 1)
+
+        # store spage resources
+        mupdf.pdf_dict_put(xobj1, PDF_NAME('Resources'), resources)
+    return xobj1
+
+
+def PySequence_Check(s):
+    return isinstance(s, (tuple, list))
+
+
+def PySequence_Size(s):
+    return len(s)
+
+
+# constants: error messages. These are also in extra.i.
+#
+MSG_BAD_ANNOT_TYPE = "bad annot type"
+MSG_BAD_APN = "bad or missing annot AP/N"
+MSG_BAD_ARG_INK_ANNOT = "arg must be seq of seq of float pairs"
+MSG_BAD_ARG_POINTS = "bad seq of points"
+MSG_BAD_BUFFER = "bad type: 'buffer'"
+MSG_BAD_COLOR_SEQ = "bad color sequence"
+MSG_BAD_DOCUMENT = "cannot open broken document"
+MSG_BAD_FILETYPE = "bad filetype"
+MSG_BAD_LOCATION = "bad location"
+MSG_BAD_OC_CONFIG = "bad config number"
+MSG_BAD_OC_LAYER = "bad layer number"
+MSG_BAD_OC_REF = "bad 'oc' reference"
+MSG_BAD_PAGEID = "bad page id"
+MSG_BAD_PAGENO = "bad page number(s)"
+MSG_BAD_PDFROOT = "PDF has no root"
+MSG_BAD_RECT = "rect is infinite or empty"
+MSG_BAD_TEXT = "bad type: 'text'"
+MSG_BAD_XREF = "bad xref"
+MSG_COLOR_COUNT_FAILED = "color count failed"
+MSG_FILE_OR_BUFFER = "need font file or buffer"
+MSG_FONT_FAILED = "cannot create font"
+MSG_IS_NO_ANNOT = "is no annotation"
+MSG_IS_NO_IMAGE = "is no image"
+MSG_IS_NO_PDF = "is no PDF"
+MSG_IS_NO_DICT = "object is no PDF dict"
+MSG_PIX_NOALPHA = "source pixmap has no alpha"
+MSG_PIXEL_OUTSIDE = "pixel(s) outside image"
+
+
+JM_Exc_FileDataError = 'FileDataError'
+PyExc_ValueError = 'ValueError'
+
+def RAISEPY( msg, exc):
+    #JM_Exc_CurrentException=exc
+    #fz_throw(context, FZ_ERROR_GENERIC, msg)
+    raise Exception( msg)
+
+
+def PyUnicode_DecodeRawUnicodeEscape(s, errors='strict'):
+    # FIXED: handle raw unicode escape sequences
+    if not s:
+        return ""
+    if isinstance(s, str):
+        rc = s.encode("utf8", errors=errors)
+    elif isinstance(s, bytes):
+        rc = s[:]
+    ret = rc.decode('raw_unicode_escape', errors=errors)
+    return ret
+
+
+def CheckColor(c: OptSeq):
+    if c:
+        if (
+            type(c) not in (list, tuple)
+            or len(c) not in (1, 3, 4)
+            or min(c) < 0
+            or max(c) > 1
+        ):
+            raise ValueError("need 1, 3 or 4 color components in range 0 to 1")
+
+
+def CheckFont(page: Page, fontname: str) -> tuple:
+    """Return an entry in the page's font list if reference name matches.
+    """
+    for f in page.get_fonts():
+        if f[4] == fontname:
+            return f
+
+
+def CheckFontInfo(doc: Document, xref: int) -> list:
+    """Return a font info if present in the document.
+    """
+    for f in doc.FontInfos:
+        if xref == f[0]:
+            return f
+
+
+def CheckMarkerArg(quads: typing.Any) -> tuple:
+    if CheckRect(quads):
+        r = Rect(quads)
+        return (r.quad,)
+    if CheckQuad(quads):
+        return (quads,)
+    for q in quads:
+        if not (CheckRect(q) or CheckQuad(q)):
+            raise ValueError("bad quads entry")
+    return quads
+
+
+def CheckMorph(o: typing.Any) -> bool:
+    if not bool(o):
+        return False
+    if not (type(o) in (list, tuple) and len(o) == 2):
+        raise ValueError("morph must be a sequence of length 2")
+    if not (len(o[0]) == 2 and len(o[1]) == 6):
+        raise ValueError("invalid morph param 0")
+    if not o[1][4] == o[1][5] == 0:
+        raise ValueError("invalid morph param 1")
+    return True
+
+
+def CheckParent(o: typing.Any):
+    return
+    if not hasattr(o, "parent") or o.parent is None:
+        raise ValueError(f"orphaned object {type(o)=}: parent is None")
+
+
+def CheckQuad(q: typing.Any) -> bool:
+    """Check whether an object is convex, not empty  quad-like.
+
+    It must be a sequence of 4 number pairs.
+    """
+    try:
+        q0 = Quad(q)
+    except Exception:
+        if g_exceptions_verbose > 1:    exception_info()
+        return False
+    return q0.is_convex
+
+
+def CheckRect(r: typing.Any) -> bool:
+    """Check whether an object is non-degenerate rect-like.
+
+    It must be a sequence of 4 numbers.
+    """
+    try:
+        r = Rect(r)
+    except Exception:
+        if g_exceptions_verbose > 1:    exception_info()
+        return False
+    return not (r.is_empty or r.is_infinite)
+
+
+def ColorCode(c: typing.Union[list, tuple, float, None], f: str) -> str:
+    if not c:
+        return ""
+    if hasattr(c, "__float__"):
+        c = (c,)
+    CheckColor(c)
+    if len(c) == 1:
+        s = _format_g(c[0]) + " "
+        return s + "G " if f == "c" else s + "g "
+
+    if len(c) == 3:
+        s = _format_g(tuple(c)) + " "
+        return s + "RG " if f == "c" else s + "rg "
+
+    s = _format_g(tuple(c)) + " "
+    return s + "K " if f == "c" else s + "k "
+
+
+def Page__add_text_marker(self, quads, annot_type):
+    pdfpage = self._pdf_page()
+    rotation = JM_page_rotation(pdfpage)
+    def final():
+        if rotation != 0:
+            mupdf.pdf_dict_put_int(pdfpage.obj(), PDF_NAME('Rotate'), rotation)
+    try:
+        if rotation != 0:
+            mupdf.pdf_dict_put_int(pdfpage.obj(), PDF_NAME('Rotate'), 0)
+        annot = mupdf.pdf_create_annot(pdfpage, annot_type)
+        for item in quads:
+            q = JM_quad_from_py(item)
+            mupdf.pdf_add_annot_quad_point(annot, q)
+        mupdf.pdf_update_annot(annot)
+        JM_add_annot_id(annot, "A")
+        final()
+    except Exception:
+        if g_exceptions_verbose:    exception_info()
+        final()
+        return
+    return Annot(annot)
+
+
+def PDF_NAME(x):
+    assert isinstance(x, str)
+    ret = getattr(mupdf, f'PDF_ENUM_NAME_{x}')
+    # Note that we return a (swig proxy for) pdf_obj*, not a mupdf.PdfObj. In
+    # the C++ API, the constructor PdfObj::PdfObj(pdf_obj*) is marked as
+    # explicit, but this seems to be ignored by SWIG. If SWIG started to
+    # generate code that respected `explicit`, we would need to do `return
+    # mupdf.PdfObj(ret)`.
+    #
+    # [Compare with extra.i, where we define our own PDF_NAME2() macro that
+    # returns a mupdf::PdfObj.]
+    return ret
+
+
+def UpdateFontInfo(doc: Document, info: typing.Sequence):
+    xref = info[0]
+    found = False
+    for i, fi in enumerate(doc.FontInfos):
+        if fi[0] == xref:
+            found = True
+            break
+    if found:
+        doc.FontInfos[i] = info
+    else:
+        doc.FontInfos.append(info)
+
+
+def args_match(args, *types):
+    '''
+    Returns true if  matches .
+
+    Each item in  is a type or tuple of types. Any of these types will
+    match an item in . `None` will match anything in . `type(None)`
+    will match an arg whose value is `None`.
+    '''
+    j = 0
+    for i in range(len(types)):
+        type_ = types[i]
+        if j >= len(args):
+            if isinstance(type_, tuple) and None in type_:
+                # arg is missing but has default value.
+                continue
+            else:
+                return False
+        if type_ is not None and not isinstance(args[j], type_):
+            return False
+        j += 1
+    if j != len(args):
+        return False
+    return True
+
+
+def calc_image_matrix(width, height, tr, rotate, keep):
+    '''
+    # compute image insertion matrix
+    '''
+    trect = JM_rect_from_py(tr)
+    rot = mupdf.fz_rotate(rotate)
+    trw = trect.x1 - trect.x0
+    trh = trect.y1 - trect.y0
+    w = trw
+    h = trh
+    if keep:
+        large = max(width, height)
+        fw = width / large
+        fh = height / large
+    else:
+        fw = fh = 1
+    small = min(fw, fh)
+    if rotate != 0 and rotate != 180:
+        f = fw
+        fw = fh
+        fh = f
+    if fw < 1:
+        if trw / fw > trh / fh:
+            w = trh * small
+            h = trh
+        else:
+            w = trw
+            h = trw / small
+    elif fw != fh:
+        if trw / fw > trh / fh:
+            w = trh / small
+            h = trh
+        else:
+            w = trw
+            h = trw * small
+    else:
+        w = trw
+        h = trh
+    tmp = mupdf.fz_make_point(
+            (trect.x0 + trect.x1) / 2,
+            (trect.y0 + trect.y1) / 2,
+            )
+    mat = mupdf.fz_make_matrix(1, 0, 0, 1, -0.5, -0.5)
+    mat = mupdf.fz_concat(mat, rot)
+    mat = mupdf.fz_concat(mat, mupdf.fz_scale(w, h))
+    mat = mupdf.fz_concat(mat, mupdf.fz_translate(tmp.x, tmp.y))
+    return mat
+
+
+def detect_super_script(line, ch):
+    if line.m_internal.wmode == 0 and line.m_internal.dir.x == 1 and line.m_internal.dir.y == 0:
+        return ch.m_internal.origin.y < line.m_internal.first_char.origin.y - ch.m_internal.size * 0.1
+    return 0
+
+
+def dir_str(x):
+    ret = f'{x} {type(x)} ({len(dir(x))}):\n'
+    for i in dir(x):
+        ret += f'    {i}\n'
+    return ret
+
+
+def getTJstr(text: str, glyphs: typing.Union[list, tuple, None], simple: bool, ordering: int) -> str:
+    """ Return a PDF string enclosed in [] brackets, suitable for the PDF TJ
+    operator.
+
+    Notes:
+        The input string is converted to either 2 or 4 hex digits per character.
+    Args:
+        simple: no glyphs: 2-chars, use char codes as the glyph
+                glyphs: 2-chars, use glyphs instead of char codes (Symbol,
+                ZapfDingbats)
+        not simple: ordering < 0: 4-chars, use glyphs not char codes
+                    ordering >=0: a CJK font! 4 chars, use char codes as glyphs
+    """
+    if text.startswith("[<") and text.endswith(">]"):  # already done
+        return text
+
+    if not bool(text):
+        return "[<>]"
+
+    if simple:  # each char or its glyph is coded as a 2-byte hex
+        if glyphs is None:  # not Symbol, not ZapfDingbats: use char code
+            otxt = "".join(["%02x" % ord(c) if ord(c) < 256 else "b7" for c in text])
+        else:  # Symbol or ZapfDingbats: use glyphs
+            otxt = "".join(
+                ["%02x" % glyphs[ord(c)][0] if ord(c) < 256 else "b7" for c in text]
+            )
+        return "[<" + otxt + ">]"
+
+    # non-simple fonts: each char or its glyph is coded as 4-byte hex
+    if ordering < 0:  # not a CJK font: use the glyphs
+        otxt = "".join(["%04x" % glyphs[ord(c)][0] for c in text])
+    else:  # CJK: use the char codes
+        otxt = "".join(["%04x" % ord(c) for c in text])
+
+    return "[<" + otxt + ">]"
+
+
+def get_pdf_str(s: str) -> str:
+    """ Return a PDF string depending on its coding.
+
+    Notes:
+        Returns a string bracketed with either "()" or "<>" for hex values.
+        If only ascii then "(original)" is returned, else if only 8 bit chars
+        then "(original)" with interspersed octal strings \nnn is returned,
+        else a string "" is returned, where [hexstring] is the
+        UTF-16BE encoding of the original.
+    """
+    if not bool(s):
+        return "()"
+
+    def make_utf16be(s):
+        r = bytearray([254, 255]) + bytearray(s, "UTF-16BE")
+        return "<" + r.hex() + ">"  # brackets indicate hex
+
+    # The following either returns the original string with mixed-in
+    # octal numbers \nnn for chars outside the ASCII range, or returns
+    # the UTF-16BE BOM version of the string.
+    r = ""
+    for c in s:
+        oc = ord(c)
+        if oc > 255:  # shortcut if beyond 8-bit code range
+            return make_utf16be(s)
+
+        if oc > 31 and oc < 127:  # in ASCII range
+            if c in ("(", ")", "\\"):  # these need to be escaped
+                r += "\\"
+            r += c
+            continue
+
+        if oc > 127:  # beyond ASCII
+            r += "\\%03o" % oc
+            continue
+
+        # now the white spaces
+        if oc == 8:  # backspace
+            r += "\\b"
+        elif oc == 9:  # tab
+            r += "\\t"
+        elif oc == 10:  # line feed
+            r += "\\n"
+        elif oc == 12:  # form feed
+            r += "\\f"
+        elif oc == 13:  # carriage return
+            r += "\\r"
+        else:
+            r += "\\267"  # unsupported: replace by 0xB7
+
+    return "(" + r + ")"
+
+
+def get_tessdata(tessdata=None):
+    """Detect Tesseract language support folder.
+
+    This function is used to enable OCR via Tesseract even if the language
+    support folder is not specified directly or in environment variable
+    TESSDATA_PREFIX.
+
+    * If  is set we return it directly.
+    
+    * Otherwise we return `os.environ['TESSDATA_PREFIX']` if set.
+    
+    * Otherwise we search for a Tesseract installation and return its language
+      support folder.
+
+    * Otherwise we raise an exception.
+    """
+    if tessdata:
+        return tessdata
+    tessdata = os.getenv("TESSDATA_PREFIX")
+    if tessdata:  # use environment variable if set
+        return tessdata
+
+    # Try to locate the tesseract-ocr installation.
+    
+    import subprocess
+    # Windows systems:
+    if sys.platform == "win32":
+        cp = subprocess.run("where tesseract", shell=1, capture_output=1, check=0, text=True)
+        response = cp.stdout.strip()
+        if cp.returncode or not response:
+            raise RuntimeError("No tessdata specified and Tesseract is not installed")
+        dirname = os.path.dirname(response)  # path of tesseract.exe
+        tessdata = os.path.join(dirname, "tessdata")  # language support
+        if os.path.exists(tessdata):  # all ok?
+            return tessdata
+        else:  # should not happen!
+            raise RuntimeError("No tessdata specified and Tesseract installation has no {tessdata} folder")
+
+    # Unix-like systems:
+    cp = subprocess.run("whereis tesseract-ocr", shell=1, capture_output=1, check=0, text=True)
+    response = cp.stdout.strip().split()
+    if cp.returncode or len(response) != 2:  # if not 2 tokens: no tesseract-ocr
+        raise RuntimeError("No tessdata specified and Tesseract is not installed")
+
+    # search tessdata in folder structure
+    dirname = response[1]  # contains tesseract-ocr installation folder
+    pattern = f"{dirname}/*/tessdata"
+    tessdatas = glob.glob(pattern)
+    tessdatas.sort()
+    if tessdatas:
+        return tessdatas[-1]
+    else:
+        raise RuntimeError("No tessdata specified and Tesseract installation has no {pattern} folder.")
+
+
+def css_for_pymupdf_font(
+    fontcode: str, *, CSS: OptStr = None, archive: AnyType = None, name: OptStr = None
+) -> str:
+    """Create @font-face items for the given fontcode of pymupdf-fonts.
+
+    Adds @font-face support for fonts contained in package pymupdf-fonts.
+
+    Creates a CSS font-family for all fonts starting with string 'fontcode'.
+
+    Note:
+        The font naming convention in package pymupdf-fonts is "fontcode",
+        where the suffix "sf" is either empty or one of "it", "bo" or "bi".
+        These suffixes thus represent the regular, italic, bold or bold-italic
+        variants of a font. For example, font code "notos" refers to fonts
+        "notos" - "Noto Sans Regular"
+        "notosit" - "Noto Sans Italic"
+        "notosbo" - "Noto Sans Bold"
+        "notosbi" - "Noto Sans Bold Italic"
+
+        This function creates four CSS @font-face definitions and collectively
+        assigns the font-family name "notos" to them (or the "name" value).
+
+    All fitting font buffers of the pymupdf-fonts package are placed / added
+    to the archive provided as parameter.
+    To use the font in pymupdf.Story, execute 'set_font(fontcode)'. The correct
+    font weight (bold) or style (italic) will automatically be selected.
+    Expects and returns the CSS source, with the new CSS definitions appended.
+
+    Args:
+        fontcode: (str) font code for naming the font variants to include.
+                  E.g. "fig" adds notos, notosi, notosb, notosbi fonts.
+                  A maximum of 4 font variants is accepted.
+        CSS: (str) CSS string to add @font-face definitions to.
+        archive: (Archive, mandatory) where to place the font buffers.
+        name: (str) use this as family-name instead of 'fontcode'.
+    Returns:
+        Modified CSS, with appended @font-face statements for each font variant
+        of fontcode.
+        Fontbuffers associated with "fontcode" will be added to 'archive'.
+    """
+    # @font-face template string
+    CSSFONT = "\n@font-face {font-family: %s; src: url(%s);%s%s}\n"
+
+    if not type(archive) is Archive:
+        raise ValueError("'archive' must be an Archive")
+    if CSS is None:
+        CSS = ""
+
+    # select font codes starting with the pass-in string
+    font_keys = [k for k in fitz_fontdescriptors.keys() if k.startswith(fontcode)]
+    if font_keys == []:
+        raise ValueError(f"No font code '{fontcode}' found in pymupdf-fonts.")
+    if len(font_keys) > 4:
+        raise ValueError("fontcode too short")
+    if name is None:  # use this name for font-family
+        name = fontcode
+
+    for fkey in font_keys:
+        font = fitz_fontdescriptors[fkey]
+        bold = font["bold"]  # determine font property
+        italic = font["italic"]  # determine font property
+        fbuff = font["loader"]()  # load the fontbuffer
+        archive.add(fbuff, fkey)  # update the archive
+        bold_text = "font-weight: bold;" if bold else ""
+        italic_text = "font-style: italic;" if italic else ""
+        CSS += CSSFONT % (name, fkey, bold_text, italic_text)
+    return CSS
+
+
+def get_text_length(text: str, fontname: str ="helv", fontsize: float =11, encoding: int =0) -> float:
+    """Calculate length of a string for a built-in font.
+
+    Args:
+        fontname: name of the font.
+        fontsize: font size points.
+        encoding: encoding to use, 0=Latin (default), 1=Greek, 2=Cyrillic.
+    Returns:
+        (float) length of text.
+    """
+    fontname = fontname.lower()
+    basename = Base14_fontdict.get(fontname, None)
+
+    glyphs = None
+    if basename == "Symbol":
+        glyphs = symbol_glyphs
+    if basename == "ZapfDingbats":
+        glyphs = zapf_glyphs
+    if glyphs is not None:
+        w = sum([glyphs[ord(c)][1] if ord(c) < 256 else glyphs[183][1] for c in text])
+        return w * fontsize
+
+    if fontname in Base14_fontdict.keys():
+        return util_measure_string(
+            text, Base14_fontdict[fontname], fontsize, encoding
+        )
+
+    if fontname in (
+        "china-t",
+        "china-s",
+        "china-ts",
+        "china-ss",
+        "japan",
+        "japan-s",
+        "korea",
+        "korea-s",
+    ):
+        return len(text) * fontsize
+
+    raise ValueError("Font '%s' is unsupported" % fontname)
+
+
+def image_profile(img: ByteString) -> dict:
+    """ Return basic properties of an image.
+
+    Args:
+        img: bytes, bytearray, io.BytesIO object or an opened image file.
+    Returns:
+        A dictionary with keys width, height, colorspace.n, bpc, type, ext and size,
+        where 'type' is the MuPDF image type (0 to 14) and 'ext' the suitable
+        file extension.
+    """
+    if type(img) is io.BytesIO:
+        stream = img.getvalue()
+    elif hasattr(img, "read"):
+        stream = img.read()
+    elif type(img) in (bytes, bytearray):
+        stream = img
+    else:
+        raise ValueError("bad argument 'img'")
+
+    return TOOLS.image_profile(stream)
+
+
+def jm_append_merge(dev):
+    '''
+    Append current path to list or merge into last path of the list.
+    (1) Append if first path, different item lists or not a 'stroke' version
+        of previous path
+    (2) If new path has the same items, merge its content into previous path
+        and change path["type"] to "fs".
+    (3) If "out" is callable, skip the previous and pass dictionary to it.
+    '''
+    #log(f'{getattr(dev, "pathdict", None)=}')
+    assert isinstance(dev.out, list)
+    #log( f'{dev.out=}')
+    
+    if callable(dev.method) or dev.method:  # function or method
+        # callback.
+        if dev.method is None:
+            # fixme, this surely cannot happen?
+            assert 0
+            #resp = PyObject_CallFunctionObjArgs(out, dev.pathdict, NULL)
+        else:
+            #log(f'calling {dev.out=} {dev.method=} {dev.pathdict=}')
+            resp = getattr(dev.out, dev.method)(dev.pathdict)
+        if not resp:
+            message("calling cdrawings callback function/method failed!")
+        dev.pathdict = None
+        return
+    
+    def append():
+        #log(f'jm_append_merge(): clearing dev.pathdict')
+        dev.out.append(dev.pathdict.copy())
+        dev.pathdict.clear()
+    assert isinstance(dev.out, list)
+    len_ = len(dev.out) # len of output list so far
+    #log('{len_=}')
+    if len_ == 0:   # always append first path
+        return append()
+    #log(f'{getattr(dev, "pathdict", None)=}')
+    thistype = dev.pathdict[ dictkey_type]
+    #log(f'{thistype=}')
+    if thistype != 's': # if not stroke, then append
+        return append()
+    prev = dev.out[ len_-1] # get prev path
+    #log( f'{prev=}')
+    prevtype = prev[ dictkey_type]
+    #log( f'{prevtype=}')
+    if prevtype != 'f': # if previous not fill, append
+        return append()
+    # last check: there must be the same list of items for "f" and "s".
+    previtems = prev[ dictkey_items]
+    thisitems = dev.pathdict[ dictkey_items]
+    if previtems != thisitems:
+        return append()
+    
+    #rc = PyDict_Merge(prev, dev.pathdict, 0);  // merge with no override
+    try:
+        for k, v in dev.pathdict.items():
+            if k not in prev:
+                prev[k] = v
+        rc = 0
+    except Exception:
+        if g_exceptions_verbose:    exception_info()
+        #raise
+        rc = -1
+    if rc == 0:
+        prev[ dictkey_type] = 'fs'
+        dev.pathdict.clear()
+    else:
+        message("could not merge stroke and fill path")
+        append()
+
+
+def jm_bbox_add_rect( dev, ctx, rect, code):
+    if not dev.layers:
+        dev.result.append( (code, JM_py_from_rect(rect)))
+    else:
+        dev.result.append( (code, JM_py_from_rect(rect), dev.layer_name))
+
+
+def jm_bbox_fill_image( dev, ctx, image, ctm, alpha, color_params):
+    r = mupdf.FzRect(mupdf.FzRect.Fixed_UNIT)
+    r = mupdf.ll_fz_transform_rect( r.internal(), ctm)
+    jm_bbox_add_rect( dev, ctx, r, "fill-image")
+
+
+def jm_bbox_fill_image_mask( dev, ctx, image, ctm, colorspace, color, alpha, color_params):
+    try:
+        jm_bbox_add_rect( dev, ctx, mupdf.ll_fz_transform_rect(mupdf.fz_unit_rect, ctm), "fill-imgmask")
+    except Exception:
+        if g_exceptions_verbose:    exception_info()
+        raise
+
+
+def jm_bbox_fill_path( dev, ctx, path, even_odd, ctm, colorspace, color, alpha, color_params):
+    even_odd = True if even_odd else False
+    try:
+        jm_bbox_add_rect( dev, ctx, mupdf.ll_fz_bound_path(path, None, ctm), "fill-path")
+    except Exception:
+        if g_exceptions_verbose:    exception_info()
+        raise
+
+
+def jm_bbox_fill_shade( dev, ctx, shade, ctm, alpha, color_params):
+    try:
+        jm_bbox_add_rect( dev, ctx, mupdf.ll_fz_bound_shade( shade, ctm), "fill-shade")
+    except Exception:
+        if g_exceptions_verbose:    exception_info()
+        raise
+
+
+def jm_bbox_stroke_text( dev, ctx, text, stroke, ctm, *args):
+    try:
+        jm_bbox_add_rect( dev, ctx, mupdf.ll_fz_bound_text( text, stroke, ctm), "stroke-text")
+    except Exception:
+        if g_exceptions_verbose:    exception_info()
+        raise
+
+
+def jm_bbox_fill_text( dev, ctx, text, ctm, *args):
+    try:
+        jm_bbox_add_rect( dev, ctx, mupdf.ll_fz_bound_text( text, None, ctm), "fill-text")
+    except Exception:
+        if g_exceptions_verbose:    exception_info()
+        raise
+
+
+def jm_bbox_ignore_text( dev, ctx, text, ctm):
+    jm_bbox_add_rect( dev, ctx, mupdf.ll_fz_bound_text(text, None, ctm), "ignore-text")
+
+
+def jm_bbox_stroke_path( dev, ctx, path, stroke, ctm, colorspace, color, alpha, color_params):
+    try:
+        jm_bbox_add_rect( dev, ctx, mupdf.ll_fz_bound_path( path, stroke, ctm), "stroke-path")
+    except Exception:
+        if g_exceptions_verbose:    exception_info()
+        raise
+
+
+def jm_checkquad(dev):
+    '''
+    Check whether the last 4 lines represent a quad.
+    Because of how we count, the lines are a polyline already, i.e. last point
+    of a line equals 1st point of next line.
+    So we check for a polygon (last line's end point equals start point).
+    If not true we return 0.
+    '''
+    #log(f'{getattr(dev, "pathdict", None)=}')
+    items = dev.pathdict[ dictkey_items]
+    len_ = len(items)
+    f = [0] * 8 # coordinates of the 4 corners
+    # fill the 8 floats in f, start from items[-4:]
+    for i in range( 4): # store line start points
+        line = items[ len_ - 4 + i]
+        temp = JM_point_from_py( line[1])
+        f[i * 2] = temp.x
+        f[i * 2 + 1] = temp.y
+        lp = JM_point_from_py( line[ 2])
+    if lp.x != f[0] or lp.y != f[1]:
+        # not a polygon!
+        #dev.linecount -= 1
+        return 0
+    
+    # we have detected a quad
+    dev.linecount = 0   # reset this
+    # a quad item is ("qu", (ul, ur, ll, lr)), where the tuple items
+    # are pairs of floats representing a quad corner each.
+    
+    # relationship of float array to quad points:
+    # (0, 1) = ul, (2, 3) = ll, (6, 7) = ur, (4, 5) = lr
+    q = mupdf.fz_make_quad(f[0], f[1], f[6], f[7], f[2], f[3], f[4], f[5])
+    rect = ('qu', JM_py_from_quad(q))
+    
+    items[ len_ - 4] = rect  # replace item -4 by rect
+    del items[ len_ - 3 : len_]  # delete remaining 3 items
+    return 1
+
+
+def jm_checkrect(dev):
+    '''
+    Check whether the last 3 path items represent a rectangle.
+    Returns 1 if we have modified the path, otherwise 0.
+    '''
+    #log(f'{getattr(dev, "pathdict", None)=}')
+    dev.linecount = 0   # reset line count
+    orientation = 0 # area orientation of rectangle
+    items = dev.pathdict[ dictkey_items]
+    len_ = len(items)
+
+    line0 = items[ len_ - 3]
+    ll = JM_point_from_py( line0[ 1])
+    lr = JM_point_from_py( line0[ 2])
+
+    # no need to extract "line1"!
+    line2 = items[ len_ - 1]
+    ur = JM_point_from_py( line2[ 1])
+    ul = JM_point_from_py( line2[ 2])
+
+    # Assumption:
+    # When decomposing rects, MuPDF always starts with a horizontal line,
+    # followed by a vertical line, followed by a horizontal line.
+    # First line: (ll, lr), third line: (ul, ur).
+    # If 1st line is below 3rd line, we record anti-clockwise (+1), else
+    # clockwise (-1) orientation.
+    
+    if (0
+            or ll.y != lr.y
+            or ll.x != ul.x
+            or ur.y != ul.y
+            or ur.x != lr.x
+            ):
+        return 0 # not a rectangle
+    
+    # we have a rect, replace last 3 "l" items by one "re" item.
+    if ul.y < lr.y:
+        r = mupdf.fz_make_rect(ul.x, ul.y, lr.x, lr.y)
+        orientation = 1
+    else:
+        r = mupdf.fz_make_rect(ll.x, ll.y, ur.x, ur.y)
+        orientation = -1
+    
+    rect = ( 're', JM_py_from_rect(r), orientation)
+    items[ len_ - 3] = rect # replace item -3 by rect
+    del items[ len_ - 2 : len_] # delete remaining 2 items
+    return 1
+
+
+def jm_trace_text( dev, text, type_, ctm, colorspace, color, alpha, seqno):
+    span = text.head
+    while 1:
+        if not span:
+            break
+        jm_trace_text_span( dev, span, type_, ctm, colorspace, color, alpha, seqno)
+        span = span.next
+
+
+def jm_trace_text_span(dev, span, type_, ctm, colorspace, color, alpha, seqno):
+    '''
+    jm_trace_text_span(fz_context *ctx, PyObject *out, fz_text_span *span, int type, fz_matrix ctm, fz_colorspace *colorspace, const float *color, float alpha, size_t seqno)
+    '''
+    out_font = None
+    assert isinstance( span, mupdf.fz_text_span)
+    span = mupdf.FzTextSpan( span)
+    assert isinstance( ctm, mupdf.fz_matrix)
+    ctm = mupdf.FzMatrix( ctm)
+    fontname = JM_font_name( span.font())
+    #float rgb[3];
+    #PyObject *chars = PyTuple_New(span->len);
+    
+    mat = mupdf.fz_concat(span.trm(), ctm)  # text transformation matrix
+    dir = mupdf.fz_transform_vector(mupdf.fz_make_point(1, 0), mat) # writing direction
+    fsize = math.sqrt(dir.x * dir.x + dir.y * dir.y)  # font size
+
+    dir = mupdf.fz_normalize_vector(dir)
+
+    space_adv = 0
+    asc = JM_font_ascender( span.font())
+    dsc = JM_font_descender( span.font())
+    if asc < 1e-3:  # probably Tesseract font
+        dsc = -0.1
+        asc = 0.9
+
+    # compute effective ascender / descender
+    ascsize = asc * fsize / (asc - dsc)
+    dscsize = dsc * fsize / (asc - dsc)
+    fflags = 0  # font flags
+    mono = mupdf.fz_font_is_monospaced( span.font())
+    fflags += mono * TEXT_FONT_MONOSPACED
+    fflags += mupdf.fz_font_is_italic( span.font()) * TEXT_FONT_ITALIC
+    fflags += mupdf.fz_font_is_serif( span.font()) * TEXT_FONT_SERIFED
+    fflags += mupdf.fz_font_is_bold( span.font()) * TEXT_FONT_BOLD
+
+    last_adv = 0
+
+    # walk through characters of span
+    span_bbox = mupdf.FzRect()
+    rot = mupdf.fz_make_matrix(dir.x, dir.y, -dir.y, dir.x, 0, 0)
+    if dir.x == -1: # left-right flip
+        rot.d = 1
+
+    chars = []
+    for i in range( span.m_internal.len):
+        adv = 0
+        if span.items(i).gid >= 0:
+            adv = mupdf.fz_advance_glyph( span.font(), span.items(i).gid, span.m_internal.wmode)
+        adv *= fsize
+        last_adv = adv
+        if span.items(i).ucs == 32:
+            space_adv = adv
+        char_orig = mupdf.fz_make_point(span.items(i).x, span.items(i).y)
+        char_orig = mupdf.fz_transform_point(char_orig, ctm)
+        m1 = mupdf.fz_make_matrix(1, 0, 0, 1, -char_orig.x, -char_orig.y)
+        m1 = mupdf.fz_concat(m1, rot)
+        m1 = mupdf.fz_concat(m1, mupdf.FzMatrix(1, 0, 0, 1, char_orig.x, char_orig.y))
+        x0 = char_orig.x
+        x1 = x0 + adv
+        if (
+                (mat.d > 0 and (dir.x == 1 or dir.x == -1))
+                or
+                (mat.b != 0 and mat.b == -mat.c)
+                ):  # up-down flip
+            y0 = char_orig.y + dscsize
+            y1 = char_orig.y + ascsize
+        else:
+            y0 = char_orig.y - ascsize
+            y1 = char_orig.y - dscsize
+        char_bbox = mupdf.fz_make_rect(x0, y0, x1, y1)
+        char_bbox = mupdf.fz_transform_rect(char_bbox, m1)
+        chars.append(
+                (
+                    span.items(i).ucs,
+                    span.items(i).gid,
+                    (
+                        char_orig.x,
+                        char_orig.y,
+                    ),
+                    (
+                        char_bbox.x0,
+                        char_bbox.y0,
+                        char_bbox.x1,
+                        char_bbox.y1,
+                    ),
+                )
+                )
+        if i > 0:
+            span_bbox = mupdf.fz_union_rect(span_bbox, char_bbox)
+        else:
+            span_bbox = char_bbox
+    chars = tuple(chars)
+    
+    if not space_adv:
+        if not (fflags & TEXT_FONT_MONOSPACED):
+            c, out_font = mupdf.fz_encode_character_with_fallback( span.font(), 32, 0, 0)
+            space_adv = mupdf.fz_advance_glyph(
+                    span.font(),
+                    c,
+                    span.m_internal.wmode,
+                    )
+            space_adv *= fsize
+            if not space_adv:
+                space_adv = last_adv
+        else:
+            space_adv = last_adv    # for mono, any char width suffices
+
+    # make the span dictionary
+    span_dict = dict()
+    span_dict[ 'dir'] = JM_py_from_point(dir)
+    span_dict[ 'font'] = JM_EscapeStrFromStr(fontname)
+    span_dict[ 'wmode'] = span.m_internal.wmode
+    span_dict[ 'flags'] =fflags
+    span_dict[ "bidi_lvl"] =span.m_internal.bidi_level
+    span_dict[ "bidi_dir"] = span.m_internal.markup_dir
+    span_dict[ 'ascender'] = asc
+    span_dict[ 'descender'] = dsc
+    span_dict[ 'colorspace'] = 3
+    
+    if colorspace:
+        rgb = mupdf.fz_convert_color(
+                mupdf.FzColorspace( mupdf.ll_fz_keep_colorspace( colorspace)),
+                color,
+                mupdf.fz_device_rgb(),
+                mupdf.FzColorspace(),
+                mupdf.FzColorParams(),
+                )
+        rgb = rgb[:3]   # mupdf.fz_convert_color() always returns 4 items.
+    else:
+        rgb = (0, 0, 0)
+    
+    if dev.linewidth > 0:   # width of character border
+        linewidth = dev.linewidth
+    else:
+        linewidth = fsize * 0.05    # default: 5% of font size
+    #log(f'{dev.linewidth=:.4f} {fsize=:.4f} {linewidth=:.4f}')
+    
+    span_dict[ 'color'] = rgb
+    span_dict[ 'size'] = fsize
+    span_dict[ "opacity"] = alpha
+    span_dict[ "linewidth"] = linewidth
+    span_dict[ "spacewidth"] = space_adv
+    span_dict[ 'type'] = type_
+    span_dict[ 'bbox'] = JM_py_from_rect(span_bbox)
+    span_dict[ 'layer'] = dev.layer_name
+    span_dict[ "seqno"] = seqno
+    span_dict[ 'chars'] = chars
+    #log(f'{span_dict=}')
+    dev.out.append( span_dict)
+
+
+def jm_lineart_color(colorspace, color):
+    #log(f' ')
+    if colorspace:
+        try:
+            # Need to be careful to use a named Python object to ensure
+            # that the `params` we pass to mupdf.ll_fz_convert_color() is
+            # valid. E.g. doing:
+            #
+            #   rgb = mupdf.ll_fz_convert_color(..., mupdf.FzColorParams().internal())
+            #
+            # - seems to end up with a corrupted `params`.
+            #
+            cs = mupdf.FzColorspace( mupdf.FzColorspace.Fixed_RGB)
+            cp = mupdf.FzColorParams()
+            rgb = mupdf.ll_fz_convert_color(
+                    colorspace,
+                    color,
+                    cs.m_internal,
+                    None,
+                    cp.internal(),
+                    )
+        except Exception:
+            if g_exceptions_verbose:    exception_info()
+            raise
+        return rgb[:3]
+    return ()
+
+
+def jm_lineart_drop_device(dev, ctx):
+    if isinstance(dev.out, list):
+        dev.out = []
+    dev.scissors = []
+ 
+ 
+def jm_lineart_fill_path( dev, ctx, path, even_odd, ctm, colorspace, color, alpha, color_params):
+    #log(f'{getattr(dev, "pathdict", None)=}')
+    #log(f'jm_lineart_fill_path(): {dev.seqno=}')
+    even_odd = True if even_odd else False
+    try:
+        assert isinstance( ctm, mupdf.fz_matrix)
+        dev.ctm = mupdf.FzMatrix( ctm)  # fz_concat(ctm, dev_ptm);
+        dev.path_type = trace_device_FILL_PATH
+        jm_lineart_path( dev, ctx, path)
+        if dev.pathdict is None:
+            return
+        #item_count = len(dev.pathdict[ dictkey_items])
+        #if item_count == 0:
+        #    return
+        dev.pathdict[ dictkey_type] ="f"
+        dev.pathdict[ "even_odd"] = even_odd
+        dev.pathdict[ "fill_opacity"] = alpha
+        #log(f'setting dev.pathdict[ "closePath"] to false')
+        #dev.pathdict[ "closePath"] = False
+        dev.pathdict[ "fill"] = jm_lineart_color( colorspace, color)
+        dev.pathdict[ dictkey_rect] = JM_py_from_rect(dev.pathrect)
+        dev.pathdict[ "seqno"] = dev.seqno
+        #jm_append_merge(dev)
+        dev.pathdict[ 'layer'] = dev.layer_name
+        if dev.clips:
+            dev.pathdict[ 'level'] = dev.depth
+        jm_append_merge(dev)
+        dev.seqno += 1
+        #log(f'jm_lineart_fill_path() end: {getattr(dev, "pathdict", None)=}')
+    except Exception:
+        if g_exceptions_verbose:    exception_info()
+        raise
+
+
+# There are 3 text trace types:
+# 0 - fill text (PDF Tr 0)
+# 1 - stroke text (PDF Tr 1)
+# 3 - ignore text (PDF Tr 3)
+
+def jm_lineart_fill_text( dev, ctx, text, ctm, colorspace, color, alpha, color_params):
+    if 0:
+        log(f'{type(ctx)=} {ctx=}')
+        log(f'{type(dev)=} {dev=}')
+        log(f'{type(text)=} {text=}')
+        log(f'{type(ctm)=} {ctm=}')
+        log(f'{type(colorspace)=} {colorspace=}')
+        log(f'{type(color)=} {color=}')
+        log(f'{type(alpha)=} {alpha=}')
+        log(f'{type(color_params)=} {color_params=}')
+    jm_trace_text(dev, text, 0, ctm, colorspace, color, alpha, dev.seqno)
+    dev.seqno += 1
+
+
+def jm_lineart_ignore_text(dev, text, ctm):
+    #log(f'{getattr(dev, "pathdict", None)=}')
+    jm_trace_text(dev, text, 3, ctm, None, None, 1, dev.seqno)
+    dev.seqno += 1
+
+
+class Walker(mupdf.FzPathWalker2):
+
+    def __init__(self, dev):
+        super().__init__()
+        self.use_virtual_moveto()
+        self.use_virtual_lineto()
+        self.use_virtual_curveto()
+        self.use_virtual_closepath()
+        self.dev = dev
+
+    def closepath(self, ctx):    # trace_close().
+        #log(f'Walker(): {self.dev.pathdict=}')
+        try:
+            if self.dev.linecount == 3:
+                if jm_checkrect(self.dev):
+                    #log(f'end1: {self.dev.pathdict=}')
+                    return
+            self.dev.linecount = 0   # reset # of consec. lines
+
+            if self.dev.havemove:
+                if self.dev.lastpoint != self.dev.firstpoint:
+                    item = ("l", JM_py_from_point(self.dev.lastpoint),
+                                 JM_py_from_point(self.dev.firstpoint))
+                    self.dev.pathdict[dictkey_items].append(item)
+                    self.dev.lastpoint = self.dev.firstpoint
+                self.dev.pathdict["closePath"] = False
+
+            else:
+                #log('setting self.dev.pathdict[ "closePath"] to true')
+                self.dev.pathdict[ "closePath"] = True
+                #log(f'end2: {self.dev.pathdict=}')
+
+            self.dev.havemove = 0
+
+        except Exception:
+            if g_exceptions_verbose:    exception_info()
+            raise
+
+    def curveto(self, ctx, x1, y1, x2, y2, x3, y3):   # trace_curveto().
+        #log(f'Walker(): {self.dev.pathdict=}')
+        try:
+            self.dev.linecount = 0  # reset # of consec. lines
+            p1 = mupdf.fz_make_point(x1, y1)
+            p2 = mupdf.fz_make_point(x2, y2)
+            p3 = mupdf.fz_make_point(x3, y3)
+            p1 = mupdf.fz_transform_point(p1, self.dev.ctm)
+            p2 = mupdf.fz_transform_point(p2, self.dev.ctm)
+            p3 = mupdf.fz_transform_point(p3, self.dev.ctm)
+            self.dev.pathrect = mupdf.fz_include_point_in_rect(self.dev.pathrect, p1)
+            self.dev.pathrect = mupdf.fz_include_point_in_rect(self.dev.pathrect, p2)
+            self.dev.pathrect = mupdf.fz_include_point_in_rect(self.dev.pathrect, p3)
+
+            list_ = (
+                    "c",
+                    JM_py_from_point(self.dev.lastpoint),
+                    JM_py_from_point(p1),
+                    JM_py_from_point(p2),
+                    JM_py_from_point(p3),
+                    )
+            self.dev.lastpoint = p3
+            self.dev.pathdict[ dictkey_items].append( list_)
+        except Exception:
+            if g_exceptions_verbose:    exception_info()
+            raise
+
+    def lineto(self, ctx, x, y):   # trace_lineto().
+        #log(f'Walker(): {self.dev.pathdict=}')
+        try:
+            p1 = mupdf.fz_transform_point( mupdf.fz_make_point(x, y), self.dev.ctm)
+            self.dev.pathrect = mupdf.fz_include_point_in_rect( self.dev.pathrect, p1)
+            list_ = (
+                    'l',
+                    JM_py_from_point( self.dev.lastpoint),
+                    JM_py_from_point(p1),
+                    )
+            self.dev.lastpoint = p1
+            items = self.dev.pathdict[ dictkey_items]
+            items.append( list_)
+            self.dev.linecount += 1 # counts consecutive lines
+            if self.dev.linecount == 4 and self.dev.path_type != trace_device_FILL_PATH:
+                # shrink to "re" or "qu" item
+                jm_checkquad(self.dev)
+        except Exception:
+            if g_exceptions_verbose:    exception_info()
+            raise
+
+    def moveto(self, ctx, x, y):   # trace_moveto().
+        if 0 and isinstance(self.dev.pathdict, dict):
+            log(f'self.dev.pathdict:')
+            for n, v in self.dev.pathdict.items():
+                log( '    {type(n)=} {len(n)=} {n!r} {n}: {v!r}: {v}')
+
+        #log(f'Walker(): {type(self.dev.pathdict)=} {self.dev.pathdict=}')
+
+        try:
+            #log( '{=dev.ctm type(dev.ctm)}')
+            self.dev.lastpoint = mupdf.fz_transform_point(
+                    mupdf.fz_make_point(x, y),
+                    self.dev.ctm,
+                    )
+            if mupdf.fz_is_infinite_rect( self.dev.pathrect):
+                self.dev.pathrect = mupdf.fz_make_rect(
+                        self.dev.lastpoint.x,
+                        self.dev.lastpoint.y,
+                        self.dev.lastpoint.x,
+                        self.dev.lastpoint.y,
+                        )
+            self.dev.firstpoint = self.dev.lastpoint
+            self.dev.havemove = 1
+            self.dev.linecount = 0  # reset # of consec. lines
+        except Exception:
+            if g_exceptions_verbose:    exception_info()
+            raise
+
+
+def jm_lineart_path(dev, ctx, path):
+    '''
+    Create the "items" list of the path dictionary
+    * either create or empty the path dictionary
+    * reset the end point of the path
+    * reset count of consecutive lines
+    * invoke fz_walk_path(), which create the single items
+    * if no items detected, empty path dict again
+    '''
+    #log(f'{getattr(dev, "pathdict", None)=}')
+    try:
+        dev.pathrect = mupdf.FzRect( mupdf.FzRect.Fixed_INFINITE)
+        dev.linecount = 0
+        dev.lastpoint = mupdf.FzPoint( 0, 0)
+        dev.pathdict = dict()
+        dev.pathdict[ dictkey_items] = []
+        
+        # First time we create a Walker instance is slow, e.g. 0.3s, then later
+        # times run in around 0.01ms. If Walker is defined locally instead of
+        # globally, each time takes 0.3s.
+        #
+        walker = Walker(dev)
+        # Unlike fz_run_page(), fz_path_walker callbacks are not passed
+        # a pointer to the struct, instead they get an arbitrary
+        # void*. The underlying C++ Director callbacks use this void* to
+        # identify the fz_path_walker instance so in turn we need to pass
+        # arg=walker.m_internal.
+        mupdf.fz_walk_path( mupdf.FzPath(mupdf.ll_fz_keep_path(path)), walker, walker.m_internal)
+        # Check if any items were added ...
+        if not dev.pathdict[ dictkey_items]:
+            dev.pathdict = None
+    except Exception:
+        if g_exceptions_verbose:    exception_info()
+        raise
+
+
+def jm_lineart_stroke_path( dev, ctx, path, stroke, ctm, colorspace, color, alpha, color_params):
+    #log(f'{dev.pathdict=} {dev.clips=}')
+    try:
+        assert isinstance( ctm, mupdf.fz_matrix)
+        dev.pathfactor = 1
+        if ctm.a != 0 and abs(ctm.a) == abs(ctm.d):
+            dev.pathfactor = abs(ctm.a)
+        elif ctm.b != 0 and abs(ctm.b) == abs(ctm.c):
+            dev.pathfactor = abs(ctm.b)
+        dev.ctm = mupdf.FzMatrix( ctm)  # fz_concat(ctm, dev_ptm);
+        dev.path_type = trace_device_STROKE_PATH
+
+        jm_lineart_path( dev, ctx, path)
+        if dev.pathdict is None:
+            return
+        dev.pathdict[ dictkey_type] = 's'
+        dev.pathdict[ 'stroke_opacity'] = alpha
+        dev.pathdict[ 'color'] = jm_lineart_color( colorspace, color)
+        dev.pathdict[ dictkey_width] = dev.pathfactor * stroke.linewidth
+        dev.pathdict[ 'lineCap'] = (
+                stroke.start_cap,
+                stroke.dash_cap,
+                stroke.end_cap,
+                )
+        dev.pathdict[ 'lineJoin'] = dev.pathfactor * stroke.linejoin
+        if 'closePath' not in dev.pathdict:
+            #log('setting dev.pathdict["closePath"] to false')
+            dev.pathdict['closePath'] = False
+
+        # output the "dashes" string
+        if stroke.dash_len:
+            buff = mupdf.fz_new_buffer( 256)
+            mupdf.fz_append_string( buff, "[ ") # left bracket
+            for i in range( stroke.dash_len):
+                # We use mupdf python's SWIG-generated floats_getitem() fn to
+                # access float *stroke.dash_list[].
+                value = mupdf.floats_getitem( stroke.dash_list, i)  # stroke.dash_list[i].
+                mupdf.fz_append_string( buff, f'{_format_g(dev.pathfactor * value)} ')
+            mupdf.fz_append_string( buff, f'] {_format_g(dev.pathfactor * stroke.dash_phase)}')
+            dev.pathdict[ 'dashes'] = buff
+        else:
+            dev.pathdict[ 'dashes'] = '[] 0'
+        dev.pathdict[ dictkey_rect] = JM_py_from_rect(dev.pathrect)
+        dev.pathdict['layer'] = dev.layer_name
+        dev.pathdict[ 'seqno'] = dev.seqno
+        if dev.clips:
+            dev.pathdict[ 'level'] = dev.depth
+        jm_append_merge(dev)
+        dev.seqno += 1
+    
+    except Exception:
+        if g_exceptions_verbose:    exception_info()
+        raise
+
+
+def jm_lineart_clip_path(dev, ctx, path, even_odd, ctm, scissor):
+    if not dev.clips:
+        return
+    dev.ctm = mupdf.FzMatrix(ctm)    # fz_concat(ctm, trace_device_ptm);
+    dev.path_type = trace_device_CLIP_PATH
+    jm_lineart_path(dev, ctx, path)
+    if dev.pathdict is None:
+        return
+    dev.pathdict[ dictkey_type] = 'clip'
+    dev.pathdict[ 'even_odd'] = bool(even_odd)
+    if 'closePath' not in dev.pathdict:
+        #log(f'setting dev.pathdict["closePath"] to False')
+        dev.pathdict['closePath'] = False
+   
+    dev.pathdict['scissor'] = JM_py_from_rect(compute_scissor(dev))
+    dev.pathdict['level'] = dev.depth
+    dev.pathdict['layer'] = dev.layer_name
+    jm_append_merge(dev)
+    dev.depth += 1
+
+
+def jm_lineart_clip_stroke_path(dev, ctx, path, stroke, ctm, scissor):
+    if not dev.clips:
+        return
+    dev.ctm = mupdf.FzMatrix(ctm)    # fz_concat(ctm, trace_device_ptm);
+    dev.path_type = trace_device_CLIP_STROKE_PATH
+    jm_lineart_path(dev, ctx, path)
+    if dev.pathdict is None:
+        return
+    dev.pathdict['dictkey_type'] = 'clip'
+    dev.pathdict['even_odd'] = None
+    if 'closePath' not in dev.pathdict:
+        #log(f'setting dev.pathdict["closePath"] to False')
+        dev.pathdict['closePath'] = False
+    dev.pathdict['scissor'] = JM_py_from_rect(compute_scissor(dev))
+    dev.pathdict['level'] = dev.depth
+    dev.pathdict['layer'] = dev.layer_name
+    jm_append_merge(dev)
+    dev.depth += 1
+
+
+def jm_lineart_clip_stroke_text(dev, ctx, text, stroke, ctm, scissor):
+    if not dev.clips:
+        return
+    compute_scissor(dev)
+    dev.depth += 1
+
+
+def jm_lineart_clip_text(dev, ctx, text, ctm, scissor):
+    if not dev.clips:
+        return
+    compute_scissor(dev)
+    dev.depth += 1
+
+
+def jm_lineart_clip_image_mask( dev, ctx, image, ctm, scissor):
+    if not dev.clips:
+        return
+    compute_scissor(dev)
+    dev.depth += 1
+ 
+
+def jm_lineart_pop_clip(dev, ctx):
+    if not dev.clips or not dev.scissors:
+        return
+    len_ = len(dev.scissors)
+    if len_ < 1:
+        return
+    del dev.scissors[-1]
+    dev.depth -= 1
+
+
+def jm_lineart_begin_layer(dev, ctx, name):
+    if name:
+        dev.layer_name = name
+    else:
+        dev.layer_name = ""
+
+
+def jm_lineart_end_layer(dev, ctx):
+    dev.layer_name = ""
+
+
+def jm_lineart_begin_group(dev, ctx, bbox, cs, isolated, knockout, blendmode, alpha):
+    #log(f'{dev.pathdict=} {dev.clips=}')
+    if not dev.clips:
+        return
+    dev.pathdict = { # Py_BuildValue("{s:s,s:N,s:N,s:N,s:s,s:f,s:i,s:N}",
+            "type": "group",
+            "rect": JM_py_from_rect(bbox),
+            "isolated": bool(isolated),
+            "knockout": bool(knockout),
+            "blendmode": mupdf.fz_blendmode_name(blendmode),
+            "opacity": alpha,
+            "level": dev.depth,
+            "layer": dev.layer_name
+            }
+    jm_append_merge(dev)
+    dev.depth += 1
+
+
+def jm_lineart_end_group(dev, ctx):
+    #log(f'{dev.pathdict=} {dev.clips=}')
+    if not dev.clips:
+        return
+    dev.depth -= 1
+
+
+def jm_lineart_stroke_text(dev, ctx, text, stroke, ctm, colorspace, color, alpha, color_params):
+    jm_trace_text(dev, text, 1, ctm, colorspace, color, alpha, dev.seqno)
+    dev.seqno += 1
+
+
+def jm_dev_linewidth( dev, ctx, path, stroke, matrix, colorspace, color, alpha, color_params):
+    dev.linewidth = stroke.linewidth
+    jm_increase_seqno( dev, ctx)
+
+
+def jm_increase_seqno( dev, ctx, *vargs):
+    try:
+        dev.seqno += 1
+    except Exception:
+        if g_exceptions_verbose:    exception_info()
+        raise
+
+
+def planish_line(p1: point_like, p2: point_like) -> Matrix:
+    """Compute matrix which maps line from p1 to p2 to the x-axis, such that it
+    maintains its length and p1 * matrix = Point(0, 0).
+
+    Args:
+        p1, p2: point_like
+    Returns:
+        Matrix which maps p1 to Point(0, 0) and p2 to a point on the x axis at
+        the same distance to Point(0,0). Will always combine a rotation and a
+        transformation.
+    """
+    p1 = Point(p1)
+    p2 = Point(p2)
+    return Matrix(util_hor_matrix(p1, p2))
+
+
+class JM_image_reporter_Filter(mupdf.PdfFilterOptions2):
+    def __init__(self):
+        super().__init__()
+        self.use_virtual_image_filter()
+
+    def image_filter( self, ctx, ctm, name, image):
+        assert isinstance(ctm, mupdf.fz_matrix)
+        JM_image_filter(self, mupdf.FzMatrix(ctm), name, image)
+        if mupdf_cppyy:
+            # cppyy doesn't appear to treat returned None as nullptr,
+            # resulting in obscure 'python exception' exception.
+            return 0
+
+
+class JM_new_bbox_device_Device(mupdf.FzDevice2):
+    def __init__(self, result, layers):
+        super().__init__()
+        self.result = result
+        self.layers = layers
+        self.use_virtual_fill_path()
+        self.use_virtual_stroke_path()
+        self.use_virtual_fill_text()
+        self.use_virtual_stroke_text()
+        self.use_virtual_ignore_text()
+        self.use_virtual_fill_shade()
+        self.use_virtual_fill_image()
+        self.use_virtual_fill_image_mask()
+        
+        self.use_virtual_begin_layer()
+        self.use_virtual_end_layer()
+
+    begin_layer = jm_lineart_begin_layer
+    end_layer = jm_lineart_end_layer
+    
+    fill_path = jm_bbox_fill_path
+    stroke_path = jm_bbox_stroke_path
+    fill_text = jm_bbox_fill_text
+    stroke_text = jm_bbox_stroke_text
+    ignore_text = jm_bbox_ignore_text
+    fill_shade = jm_bbox_fill_shade
+    fill_image = jm_bbox_fill_image
+    fill_image_mask = jm_bbox_fill_image_mask
+    
+
+class JM_new_output_fileptr_Output(mupdf.FzOutput2):
+    def __init__(self, bio):
+        super().__init__()
+        self.bio = bio
+        self.use_virtual_write()
+        self.use_virtual_seek()
+        self.use_virtual_tell()
+        self.use_virtual_truncate()
+    
+    def seek( self, ctx, offset, whence):
+        return self.bio.seek( offset, whence)
+    
+    def tell( self, ctx):
+        ret = self.bio.tell()
+        return ret
+    
+    def truncate( self, ctx):
+        return self.bio.truncate()
+    
+    def write(self, ctx, data_raw, data_length):
+        data = mupdf.raw_to_python_bytes(data_raw, data_length)
+        return self.bio.write(data)
+
+
+def compute_scissor(dev):
+    '''
+    Every scissor of a clip is a sub rectangle of the preceding clip scissor
+    if the clip level is larger.
+    '''
+    if dev.scissors is None:
+        dev.scissors = list()
+    num_scissors = len(dev.scissors)
+    if num_scissors > 0:
+        last_scissor = dev.scissors[num_scissors-1]
+        scissor = JM_rect_from_py(last_scissor)
+        scissor = mupdf.fz_intersect_rect(scissor, dev.pathrect)
+    else:
+        scissor = dev.pathrect
+    dev.scissors.append(JM_py_from_rect(scissor))
+    return scissor
+
+
+class JM_new_lineart_device_Device(mupdf.FzDevice2):
+    '''
+    LINEART device for Python method Page.get_cdrawings()
+    '''
+    #log(f'JM_new_lineart_device_Device()')
+    def __init__(self, out, clips, method):
+        #log(f'JM_new_lineart_device_Device.__init__()')
+        super().__init__()
+        # fixme: this results in "Unexpected call of unimplemented virtual_fnptrs fn FzDevice2::drop_device().".
+        #self.use_virtual_drop_device()
+        self.use_virtual_fill_path()
+        self.use_virtual_stroke_path()
+        self.use_virtual_clip_path()
+        self.use_virtual_clip_image_mask()
+        self.use_virtual_clip_stroke_path()
+        self.use_virtual_clip_stroke_text()
+        self.use_virtual_clip_text()
+        
+        self.use_virtual_fill_text
+        self.use_virtual_stroke_text
+        self.use_virtual_ignore_text
+        
+        self.use_virtual_fill_shade()
+        self.use_virtual_fill_image()
+        self.use_virtual_fill_image_mask()
+        
+        self.use_virtual_pop_clip()
+        
+        self.use_virtual_begin_group()
+        self.use_virtual_end_group()
+        
+        self.use_virtual_begin_layer()
+        self.use_virtual_end_layer()
+        
+        self.out = out
+        self.seqno = 0
+        self.depth = 0
+        self.clips = clips
+        self.method = method
+        
+        self.scissors = None
+        self.layer_name = ""  # optional content name
+        self.pathrect = None
+        
+        self.linewidth = 0
+        self.ptm = mupdf.FzMatrix()
+        self.ctm = mupdf.FzMatrix()
+        self.rot = mupdf.FzMatrix()
+        self.lastpoint = mupdf.FzPoint()
+        self.firstpoint = mupdf.FzPoint()
+        self.havemove = 0
+        self.pathrect = mupdf.FzRect()
+        self.pathfactor = 0
+        self.linecount = 0
+        self.path_type = 0
+    
+    #drop_device = jm_lineart_drop_device
+    
+    fill_path           = jm_lineart_fill_path
+    stroke_path         = jm_lineart_stroke_path
+    clip_image_mask     = jm_lineart_clip_image_mask
+    clip_path           = jm_lineart_clip_path
+    clip_stroke_path    = jm_lineart_clip_stroke_path
+    clip_text           = jm_lineart_clip_text
+    clip_stroke_text    = jm_lineart_clip_stroke_text
+    
+    fill_text           = jm_increase_seqno
+    stroke_text         = jm_increase_seqno
+    ignore_text         = jm_increase_seqno
+    
+    fill_shade          = jm_increase_seqno
+    fill_image          = jm_increase_seqno
+    fill_image_mask     = jm_increase_seqno
+    
+    pop_clip            = jm_lineart_pop_clip
+    
+    begin_group         = jm_lineart_begin_group
+    end_group           = jm_lineart_end_group
+    
+    begin_layer         = jm_lineart_begin_layer
+    end_layer           = jm_lineart_end_layer
+    
+
+class JM_new_texttrace_device(mupdf.FzDevice2):
+    '''
+    Trace TEXT device for Python method Page.get_texttrace()
+    '''
+
+    def __init__(self, out):
+        super().__init__()
+        self.use_virtual_fill_path()
+        self.use_virtual_stroke_path()
+        self.use_virtual_fill_text()
+        self.use_virtual_stroke_text()
+        self.use_virtual_ignore_text()
+        self.use_virtual_fill_shade()
+        self.use_virtual_fill_image()
+        self.use_virtual_fill_image_mask()
+        
+        self.use_virtual_begin_layer()
+        self.use_virtual_end_layer()
+        
+        self.out = out
+        
+        self.seqno = 0
+        self.depth = 0
+        self.clips = 0
+        self.method = None
+        
+        self.seqno = 0
+
+        self.pathdict = dict()
+        self.scissors = list()
+        self.linewidth = 0
+        self.ptm = mupdf.FzMatrix()
+        self.ctm = mupdf.FzMatrix()
+        self.rot = mupdf.FzMatrix()
+        self.lastpoint = mupdf.FzPoint()
+        self.pathrect = mupdf.FzRect()
+        self.pathfactor = 0
+        self.linecount = 0
+        self.path_type = 0
+        self.layer_name = ""
+    
+    fill_path = jm_increase_seqno
+    stroke_path = jm_dev_linewidth
+    fill_text = jm_lineart_fill_text
+    stroke_text = jm_lineart_stroke_text
+    ignore_text = jm_lineart_ignore_text
+    fill_shade = jm_increase_seqno
+    fill_image = jm_increase_seqno
+    fill_image_mask = jm_increase_seqno
+    
+    begin_layer = jm_lineart_begin_layer
+    end_layer = jm_lineart_end_layer
+
+
+def ConversionHeader(i: str, filename: OptStr ="unknown"):
+    t = i.lower()
+    import textwrap
+    html = textwrap.dedent("""
+            
+            
+            
+            
+            
+            
+            """)
+
+    xml = textwrap.dedent("""
+            
+            
+            """
+            % filename
+            )
+
+    xhtml = textwrap.dedent("""
+            
+            
+            
+            
+            
+            
+            
+            """)
+
+    text = ""
+    json = '{"document": "%s", "pages": [\n' % filename
+    if t == "html":
+        r = html
+    elif t == "json":
+        r = json
+    elif t == "xml":
+        r = xml
+    elif t == "xhtml":
+        r = xhtml
+    else:
+        r = text
+
+    return r
+
+
+def ConversionTrailer(i: str):
+    t = i.lower()
+    text = ""
+    json = "]\n}"
+    html = "\n\n"
+    xml = "\n"
+    xhtml = html
+    if t == "html":
+        r = html
+    elif t == "json":
+        r = json
+    elif t == "xml":
+        r = xml
+    elif t == "xhtml":
+        r = xhtml
+    else:
+        r = text
+
+    return r
+
+
+def annot_preprocess(page: "Page") -> int:
+    """Prepare for annotation insertion on the page.
+
+    Returns:
+        Old page rotation value. Temporarily sets rotation to 0 when required.
+    """
+    CheckParent(page)
+    if not page.parent.is_pdf:
+        raise ValueError("is no PDF")
+    old_rotation = page.rotation
+    if old_rotation != 0:
+        page.set_rotation(0)
+    return old_rotation
+
+
+def annot_postprocess(page: "Page", annot: "Annot") -> None:
+    """Clean up after annotation insertion.
+
+    Set ownership flag and store annotation in page annotation dictionary.
+    """
+    #annot.parent = weakref.proxy(page)
+    assert isinstance( page, Page)
+    assert isinstance( annot, Annot)
+    annot.parent = page
+    page._annot_refs[id(annot)] = annot
+    annot.thisown = True
+
+
+def canon(c):
+    assert isinstance(c, int)
+    # TODO: proper unicode case folding
+    # TODO: character equivalence (a matches ä, etc)
+    if c == 0xA0 or c == 0x2028 or c == 0x2029:
+        return ord(' ')
+    if c == ord('\r') or c == ord('\n') or c == ord('\t'):
+        return ord(' ')
+    if c >= ord('A') and c <= ord('Z'):
+        return c - ord('A') + ord('a')
+    return c
+
+
+def chartocanon(s):
+    assert isinstance(s, str)
+    n, c = mupdf.fz_chartorune(s)
+    c = canon(c)
+    return n, c
+
+
+def dest_is_valid(o, page_count, page_object_nums, names_list):
+    p = mupdf.pdf_dict_get( o, PDF_NAME('A'))
+    if (
+            mupdf.pdf_name_eq(
+                mupdf.pdf_dict_get( p, PDF_NAME('S')),
+                PDF_NAME('GoTo')
+                )
+            and not string_in_names_list(
+                mupdf.pdf_dict_get( p, PDF_NAME('D')),
+                names_list
+                )
+            ):
+        return 0
+
+    p = mupdf.pdf_dict_get( o, PDF_NAME('Dest'))
+    if not p.m_internal:
+        pass
+    elif mupdf.pdf_is_string( p):
+        return string_in_names_list( p, names_list)
+    elif not dest_is_valid_page(
+            mupdf.pdf_array_get( p, 0),
+            page_object_nums,
+            page_count,
+            ):
+        return 0
+    return 1
+
+
+def dest_is_valid_page(obj, page_object_nums, pagecount):
+    num = mupdf.pdf_to_num(obj)
+
+    if num == 0:
+        return 0
+    for i in range(pagecount):
+        if page_object_nums[i] == num:
+            return 1
+    return 0
+
+
+def find_string(s, needle):
+    assert isinstance(s, str)
+    for i in range(len(s)):
+        end = match_string(s[i:], needle)
+        if end is not None:
+            end += i
+            return i, end
+    return None, None
+
+
+def get_pdf_now() -> str:
+    '''
+    "Now" timestamp in PDF Format
+    '''
+    import time
+    tz = "%s'%s'" % (
+        str(abs(time.altzone // 3600)).rjust(2, "0"),
+        str((abs(time.altzone // 60) % 60)).rjust(2, "0"),
+    )
+    tstamp = time.strftime("D:%Y%m%d%H%M%S", time.localtime())
+    if time.altzone > 0:
+        tstamp += "-" + tz
+    elif time.altzone < 0:
+        tstamp += "+" + tz
+    else:
+        pass
+    return tstamp
+
+
+class ElementPosition(object):
+    """Convert a dictionary with element position information to an object."""
+
+    def __init__(self):
+        pass
+
+
+def make_story_elpos():
+    return ElementPosition()
+
+ 
+def get_highlight_selection(page, start: point_like =None, stop: point_like =None, clip: rect_like =None) -> list:
+    """Return rectangles of text lines between two points.
+
+    Notes:
+        The default of 'start' is top-left of 'clip'. The default of 'stop'
+        is bottom-reight of 'clip'.
+
+    Args:
+        start: start point_like
+        stop: end point_like, must be 'below' start
+        clip: consider this rect_like only, default is page rectangle
+    Returns:
+        List of line bbox intersections with the area established by the
+        parameters.
+    """
+    # validate and normalize arguments
+    if clip is None:
+        clip = page.rect
+    clip = Rect(clip)
+    if start is None:
+        start = clip.tl
+    if stop is None:
+        stop = clip.br
+    clip.y0 = start.y
+    clip.y1 = stop.y
+    if clip.is_empty or clip.is_infinite:
+        return []
+
+    # extract text of page, clip only, no images, expand ligatures
+    blocks = page.get_text(
+        "dict", flags=0, clip=clip,
+    )["blocks"]
+
+    lines = []  # will return this list of rectangles
+    for b in blocks:
+        bbox = Rect(b["bbox"])
+        if bbox.is_infinite or bbox.is_empty:
+            continue
+        for line in b["lines"]:
+            bbox = Rect(line["bbox"])
+            if bbox.is_infinite or bbox.is_empty:
+                continue
+            lines.append(bbox)
+
+    if lines == []:  # did not select anything
+        return lines
+
+    lines.sort(key=lambda bbox: bbox.y1)  # sort by vertical positions
+
+    # cut off prefix from first line if start point is close to its top
+    bboxf = lines.pop(0)
+    if bboxf.y0 - start.y <= 0.1 * bboxf.height:  # close enough?
+        r = Rect(start.x, bboxf.y0, bboxf.br)  # intersection rectangle
+        if not (r.is_empty or r.is_infinite):
+            lines.insert(0, r)  # insert again if not empty
+    else:
+        lines.insert(0, bboxf)  # insert again
+
+    if lines == []:  # the list might have been emptied
+        return lines
+
+    # cut off suffix from last line if stop point is close to its bottom
+    bboxl = lines.pop()
+    if stop.y - bboxl.y1 <= 0.1 * bboxl.height:  # close enough?
+        r = Rect(bboxl.tl, stop.x, bboxl.y1)  # intersection rectangle
+        if not (r.is_empty or r.is_infinite):
+            lines.append(r)  # append if not empty
+    else:
+        lines.append(bboxl)  # append again
+
+    return lines
+
+
+def glyph_name_to_unicode(name: str) -> int:
+    """Convenience function accessing unicodedata."""
+    import unicodedata
+    try:
+        unc = ord(unicodedata.lookup(name))
+    except Exception:
+        unc = 65533
+    return unc
+
+
+def hdist(dir, a, b):
+    dx = b.x - a.x
+    dy = b.y - a.y
+    return mupdf.fz_abs(dx * dir.x + dy * dir.y)
+
+
+def make_table(rect: rect_like =(0, 0, 1, 1), cols: int =1, rows: int =1) -> list:
+    """Return a list of (rows x cols) equal sized rectangles.
+
+    Notes:
+        A utility to fill a given area with table cells of equal size.
+    Args:
+        rect: rect_like to use as the table area
+        rows: number of rows
+        cols: number of columns
+    Returns:
+        A list with  items, where each item is a list of 
+        PyMuPDF Rect objects of equal sizes.
+    """
+    rect = Rect(rect)  # ensure this is a Rect
+    if rect.is_empty or rect.is_infinite:
+        raise ValueError("rect must be finite and not empty")
+    tl = rect.tl
+
+    height = rect.height / rows  # height of one table cell
+    width = rect.width / cols  # width of one table cell
+    delta_h = (width, 0, width, 0)  # diff to next right rect
+    delta_v = (0, height, 0, height)  # diff to next lower rect
+
+    r = Rect(tl, tl.x + width, tl.y + height)  # first rectangle
+
+    # make the first row
+    row = [r]
+    for i in range(1, cols):
+        r += delta_h  # build next rect to the right
+        row.append(r)
+
+    # make result, starts with first row
+    rects = [row]
+    for i in range(1, rows):
+        row = rects[i - 1]  # take previously appended row
+        nrow = []  # the new row to append
+        for r in row:  # for each previous cell add its downward copy
+            nrow.append(r + delta_v)
+        rects.append(nrow)  # append new row to result
+
+    return rects
+
+
+def util_ensure_widget_calc(annot):
+    '''
+    Ensure that widgets with /AA/C JavaScript are in array AcroForm/CO
+    '''
+    annot_obj = mupdf.pdf_annot_obj(annot.this)
+    pdf = mupdf.pdf_get_bound_document(annot_obj)
+    PDFNAME_CO = mupdf.pdf_new_name("CO")    # = PDF_NAME(CO)
+    acro = mupdf.pdf_dict_getl(  # get AcroForm dict
+            mupdf.pdf_trailer(pdf),
+            PDF_NAME('Root'),
+            PDF_NAME('AcroForm'),
+            )
+
+    CO = mupdf.pdf_dict_get(acro, PDFNAME_CO)  # = AcroForm/CO
+    if not mupdf.pdf_is_array(CO):
+        CO = mupdf.pdf_dict_put_array(acro, PDFNAME_CO, 2)
+    n = mupdf.pdf_array_len(CO)
+    found = 0
+    xref = mupdf.pdf_to_num(annot_obj)
+    for i in range(n):
+        nxref = mupdf.pdf_to_num(mupdf.pdf_array_get(CO, i))
+        if xref == nxref:
+            found = 1
+            break
+    if not found:
+        mupdf.pdf_array_push(CO, mupdf.pdf_new_indirect(pdf, xref, 0))
+
+
+def util_make_rect( *args, p0=None, p1=None, x0=None, y0=None, x1=None, y1=None):
+    '''
+    Helper for initialising rectangle classes.
+    
+    2022-09-02: This is quite different from PyMuPDF's util_make_rect(), which
+    uses `goto` in ways that don't easily translate to Python.
+
+    Returns (x0, y0, x1, y1) derived from , then override with p0, p1,
+    x0, y0, x1, y1 if they are not None.
+
+    Accepts following forms for :
+        () returns all zeros.
+        (top-left, bottom-right)
+        (top-left, x1, y1)
+        (x0, y0, bottom-right)
+        (x0, y0, x1, y1)
+        (rect)
+
+    Where top-left and bottom-right are (x, y) or something with .x, .y
+    members; rect is something with .x0, .y0, .x1, and .y1 members.
+
+    2023-11-18: we now override with p0, p1, x0, y0, x1, y1 if not None.
+    '''
+    def get_xy( arg):
+        if isinstance( arg, (list, tuple)) and len( arg) == 2:
+            return arg[0], arg[1]
+        if isinstance( arg, (Point, mupdf.FzPoint, mupdf.fz_point)):
+            return arg.x, arg.y
+        return None, None
+    def make_tuple( a):
+        if isinstance( a, tuple):
+            return a
+        if isinstance( a, Point):
+            return a.x, a.y
+        elif isinstance( a, (Rect, IRect, mupdf.FzRect, mupdf.fz_rect)):
+            return a.x0, a.y0, a.x1, a.y1
+        if not isinstance( a, (list, tuple)):
+            a = a,
+        return a
+    def handle_args():
+        if len(args) == 0:
+            return 0, 0, 0, 0
+        elif len(args) == 1:
+            arg = args[0]
+            if isinstance( arg, (list, tuple)) and len( arg) == 2:
+                p1, p2 = arg
+                ret = *p1, *p2
+                assert len(ret) == 4
+                return ret
+            if isinstance( arg, (list, tuple)) and len( arg) == 3:
+                a, b, c = arg
+                a = make_tuple(a)
+                b = make_tuple(b)
+                c = make_tuple(c)
+                ret = *a, *b, *c
+                assert len(ret) == 4
+                return ret
+            ret = make_tuple( arg)
+            assert len(ret) == 4, f'{arg=} {ret=}'
+            return ret
+        elif len(args) == 2:
+            ret = get_xy( args[0]) + get_xy( args[1])
+            assert len(ret) == 4
+            return ret
+        elif len(args) == 3:
+            x0, y0 = get_xy( args[0])
+            if (x0, y0) != (None, None):
+                return x0, y0, args[1], args[2]
+            x1, y1 = get_xy( args[2])
+            if (x1, y1) != (None, None):
+                return args[0], args[1], x1, y1
+        elif len(args) == 4:
+            return args[0], args[1], args[2], args[3]
+        raise Exception( f'Unrecognised args: {args}')
+    ret_x0, ret_y0, ret_x1, ret_y1 = handle_args()
+    if p0 is not None:  ret_x0, ret_y0 = get_xy(p0)
+    if p1 is not None:  ret_x1, ret_y1 = get_xy(p1)
+    if x0 is not None:  ret_x0 = x0
+    if y0 is not None:  ret_y0 = y0
+    if x1 is not None:  ret_x1 = x1
+    if y1 is not None:  ret_y1 = y1
+    return ret_x0, ret_y0, ret_x1, ret_y1
+
+
+def util_make_irect( *args, p0=None, p1=None, x0=None, y0=None, x1=None, y1=None):
+    a, b, c, d = util_make_rect( *args, p0=p0, p1=p1, x0=x0, y0=y0, x1=x1, y1=y1)
+    def convert(x, ceil):
+        if ceil:
+            return int(math.ceil(x))
+        else:
+            return int(math.floor(x))
+    a = convert(a, False)
+    b = convert(b, False)
+    c = convert(c, True)
+    d = convert(d, True)
+    return a, b, c, d
+
+
+def util_round_rect( rect):
+    return JM_py_from_irect(mupdf.fz_round_rect(JM_rect_from_py(rect)))
+
+
+def util_transform_rect( rect, matrix):
+    if g_use_extra:
+        return extra.util_transform_rect( rect, matrix)
+    return JM_py_from_rect(mupdf.fz_transform_rect(JM_rect_from_py(rect), JM_matrix_from_py(matrix)))
+
+
+def util_intersect_rect( r1, r2):
+    return JM_py_from_rect(
+            mupdf.fz_intersect_rect(
+                JM_rect_from_py(r1),
+                JM_rect_from_py(r2),
+                )
+            )
+
+
+def util_is_point_in_rect( p, r):
+    return mupdf.fz_is_point_inside_rect(
+                JM_point_from_py(p),
+                JM_rect_from_py(r),
+                )
+
+def util_include_point_in_rect( r, p):
+    return JM_py_from_rect(
+            mupdf.fz_include_point_in_rect(
+                JM_rect_from_py(r),
+                JM_point_from_py(p),
+                )
+            )
+
+
+def util_point_in_quad( P, Q):
+    p = JM_point_from_py(P)
+    q = JM_quad_from_py(Q)
+    return mupdf.fz_is_point_inside_quad(p, q)
+
+
+def util_transform_point( point, matrix):
+    return JM_py_from_point(
+            mupdf.fz_transform_point(
+                JM_point_from_py(point),
+                JM_matrix_from_py(matrix),
+                )
+            )
+
+
+def util_union_rect( r1, r2):
+    return JM_py_from_rect(
+            mupdf.fz_union_rect(
+                JM_rect_from_py(r1),
+                JM_rect_from_py(r2),
+                )
+            )
+
+
+def util_concat_matrix( m1, m2):
+    return JM_py_from_matrix(
+            mupdf.fz_concat(
+                JM_matrix_from_py(m1),
+                JM_matrix_from_py(m2),
+                )
+            )
+
+
+def util_invert_matrix(matrix):
+    if 0:
+        # Use MuPDF's fz_invert_matrix().
+        if isinstance( matrix, (tuple, list)):
+            matrix = mupdf.FzMatrix( *matrix)
+        elif isinstance( matrix, mupdf.fz_matrix):
+            matrix = mupdf.FzMatrix( matrix)
+        elif isinstance( matrix, Matrix):
+            matrix = mupdf.FzMatrix( matrix.a, matrix.b, matrix.c, matrix.d, matrix.e, matrix.f)
+        assert isinstance( matrix, mupdf.FzMatrix), f'{type(matrix)=}: {matrix}'
+        ret = mupdf.fz_invert_matrix( matrix)
+        if ret == matrix and (0
+                or abs( matrix.a - 1) >= sys.float_info.epsilon
+                or abs( matrix.b - 0) >= sys.float_info.epsilon
+                or abs( matrix.c - 0) >= sys.float_info.epsilon
+                or abs( matrix.d - 1) >= sys.float_info.epsilon
+                ):
+            # Inversion not possible.
+            return 1, ()
+        return 0, (ret.a, ret.b, ret.c, ret.d, ret.e, ret.f)
+    # Do inversion in python.
+    src = JM_matrix_from_py(matrix)
+    a = src.a
+    det = a * src.d - src.b * src.c
+    if det < -sys.float_info.epsilon or det > sys.float_info.epsilon:
+        dst = mupdf.FzMatrix()
+        rdet = 1 / det
+        dst.a = src.d * rdet
+        dst.b = -src.b * rdet
+        dst.c = -src.c * rdet
+        dst.d = a * rdet
+        a = -src.e * dst.a - src.f * dst.c
+        dst.f = -src.e * dst.b - src.f * dst.d
+        dst.e = a
+        return 0, (dst.a, dst.b, dst.c, dst.d, dst.e, dst.f)
+
+    return 1, ()
+
+
+def util_measure_string( text, fontname, fontsize, encoding):
+    font = mupdf.fz_new_base14_font(fontname)
+    w = 0
+    pos = 0
+    while pos < len(text):
+        t, c = mupdf.fz_chartorune(text[pos:])
+        pos += t
+        if encoding == mupdf.PDF_SIMPLE_ENCODING_GREEK:
+            c = mupdf.fz_iso8859_7_from_unicode(c)
+        elif encoding == mupdf.PDF_SIMPLE_ENCODING_CYRILLIC:
+            c = mupdf.fz_windows_1251_from_unicode(c)
+        else:
+            c = mupdf.fz_windows_1252_from_unicode(c)
+        if c < 0:
+            c = 0xB7
+        g = mupdf.fz_encode_character(font, c)
+        dw = mupdf.fz_advance_glyph(font, g, 0)
+        w += dw
+    ret = w * fontsize
+    return ret
+
+
+def util_sine_between(C, P, Q):
+    # for points C, P, Q compute the sine between lines CP and QP
+    c = JM_point_from_py(C)
+    p = JM_point_from_py(P)
+    q = JM_point_from_py(Q)
+    s = mupdf.fz_normalize_vector(mupdf.fz_make_point(q.x - p.x, q.y - p.y))
+    m1 = mupdf.fz_make_matrix(1, 0, 0, 1, -p.x, -p.y)
+    m2 = mupdf.fz_make_matrix(s.x, -s.y, s.y, s.x, 0, 0)
+    m1 = mupdf.fz_concat(m1, m2)
+    c = mupdf.fz_transform_point(c, m1)
+    c = mupdf.fz_normalize_vector(c)
+    return c.y
+
+
+def util_hor_matrix(C, P):
+    '''
+    Return the matrix that maps two points C, P to the x-axis such that
+    C -> (0,0) and the image of P have the same distance.
+    '''
+    c = JM_point_from_py(C)
+    p = JM_point_from_py(P)
+    
+    # compute (cosine, sine) of vector P-C with double precision:
+    s = mupdf.fz_normalize_vector(mupdf.fz_make_point(p.x - c.x, p.y - c.y))
+    
+    m1 = mupdf.fz_make_matrix(1, 0, 0, 1, -c.x, -c.y)
+    m2 = mupdf.fz_make_matrix(s.x, -s.y, s.y, s.x, 0, 0)
+    return JM_py_from_matrix(mupdf.fz_concat(m1, m2))
+
+
+def match_string(h0, n0):
+    h = 0
+    n = 0
+    e = h
+    delta_h, hc = chartocanon(h0[h:])
+    h += delta_h
+    delta_n, nc = chartocanon(n0[n:])
+    n += delta_n
+    while hc == nc:
+        e = h
+        if hc == ord(' '):
+            while 1:
+                delta_h, hc = chartocanon(h0[h:])
+                h += delta_h
+                if hc != ord(' '):
+                    break
+        else:
+            delta_h, hc = chartocanon(h0[h:])
+            h += delta_h
+        if nc == ord(' '):
+            while 1:
+                delta_n, nc = chartocanon(n0[n:])
+                n += delta_n
+                if nc != ord(' '):
+                    break
+        else:
+            delta_n, nc = chartocanon(n0[n:])
+            n += delta_n
+    return None if nc != 0 else e
+
+
+def on_highlight_char(hits, line, ch):
+    assert hits
+    assert isinstance(line, mupdf.FzStextLine)
+    assert isinstance(ch, mupdf.FzStextChar)
+    vfuzz = ch.m_internal.size * hits.vfuzz
+    hfuzz = ch.m_internal.size * hits.hfuzz
+    ch_quad = JM_char_quad(line, ch)
+    if hits.len > 0:
+        # fixme: end = hits.quads[-1]
+        quad = hits.quads[hits.len - 1]
+        end = JM_quad_from_py(quad)
+        if ( 1
+                and hdist(line.m_internal.dir, end.lr, ch_quad.ll) < hfuzz
+                and vdist(line.m_internal.dir, end.lr, ch_quad.ll) < vfuzz
+                and hdist(line.m_internal.dir, end.ur, ch_quad.ul) < hfuzz
+                and vdist(line.m_internal.dir, end.ur, ch_quad.ul) < vfuzz
+                ):
+            end.ur = ch_quad.ur
+            end.lr = ch_quad.lr
+            assert hits.quads[-1] == end
+            return
+    hits.quads.append(ch_quad)
+    hits.len += 1
+
+
+def page_merge(doc_des, doc_src, page_from, page_to, rotate, links, copy_annots, graft_map):
+    '''
+    Deep-copies a source page to the target.
+    Modified version of function of pdfmerge.c: we also copy annotations, but
+    we skip some subtypes. In addition we rotate output.
+    '''
+    if g_use_extra:
+        #log( 'Calling C++ extra.page_merge()')
+        return extra.page_merge( doc_des, doc_src, page_from, page_to, rotate, links, copy_annots, graft_map)
+    
+    # list of object types (per page) we want to copy
+    known_page_objs = [
+        PDF_NAME('Contents'),
+        PDF_NAME('Resources'),
+        PDF_NAME('MediaBox'),
+        PDF_NAME('CropBox'),
+        PDF_NAME('BleedBox'),
+        PDF_NAME('TrimBox'),
+        PDF_NAME('ArtBox'),
+        PDF_NAME('Rotate'),
+        PDF_NAME('UserUnit'),
+        ]
+    page_ref = mupdf.pdf_lookup_page_obj(doc_src, page_from)
+
+    # make new page dict in dest doc
+    page_dict = mupdf.pdf_new_dict(doc_des, 4)
+    mupdf.pdf_dict_put(page_dict, PDF_NAME('Type'), PDF_NAME('Page'))
+
+    # copy objects of source page into it
+    for i in range( len(known_page_objs)):
+        obj = mupdf.pdf_dict_get_inheritable( page_ref, known_page_objs[i])
+        if obj.m_internal:
+            #log( '{=type(graft_map) type(graft_map.this)}')
+            mupdf.pdf_dict_put( page_dict, known_page_objs[i], mupdf.pdf_graft_mapped_object(graft_map.this, obj))
+
+    # Copy annotations, but skip Link, Popup, IRT, Widget types
+    # If selected, remove dict keys P (parent) and Popup
+    if copy_annots:
+        old_annots = mupdf.pdf_dict_get( page_ref, PDF_NAME('Annots'))
+        n = mupdf.pdf_array_len( old_annots)
+        if n > 0:
+            new_annots = mupdf.pdf_dict_put_array( page_dict, PDF_NAME('Annots'), n)
+            for i in range(n):
+                o = mupdf.pdf_array_get( old_annots, i)
+                if not o.m_internal or not mupdf.pdf_is_dict(o):
+                    continue    # skip non-dict items
+                if mupdf.pdf_dict_gets( o, "IRT").m_internal:
+                    continue
+                subtype = mupdf.pdf_dict_get( o, PDF_NAME('Subtype'))
+                if mupdf.pdf_name_eq( subtype, PDF_NAME('Link')):
+                    continue
+                if mupdf.pdf_name_eq( subtype, PDF_NAME('Popup')):
+                    continue
+                if mupdf.pdf_name_eq(subtype, PDF_NAME('Widget')):
+                    continue
+                mupdf.pdf_dict_del( o, PDF_NAME('Popup'))
+                mupdf.pdf_dict_del( o, PDF_NAME('P'))
+                copy_o = mupdf.pdf_graft_mapped_object( graft_map.this, o)
+                annot = mupdf.pdf_new_indirect( doc_des, mupdf.pdf_to_num( copy_o), 0)
+                mupdf.pdf_array_push( new_annots, annot)
+
+    # rotate the page
+    if rotate != -1:
+        mupdf.pdf_dict_put_int( page_dict, PDF_NAME('Rotate'), rotate)
+    # Now add the page dictionary to dest PDF
+    ref = mupdf.pdf_add_object( doc_des, page_dict)
+
+    # Insert new page at specified location
+    mupdf.pdf_insert_page( doc_des, page_to, ref)
+
+
+def paper_rect(s: str) -> Rect:
+    """Return a Rect for the paper size indicated in string 's'. Must conform to the argument of method 'PaperSize', which will be invoked.
+    """
+    width, height = paper_size(s)
+    return Rect(0.0, 0.0, width, height)
+
+
+def paper_size(s: str) -> tuple:
+    """Return a tuple (width, height) for a given paper format string.
+
+    Notes:
+        'A4-L' will return (842, 595), the values for A4 landscape.
+        Suffix '-P' and no suffix return the portrait tuple.
+    """
+    size = s.lower()
+    f = "p"
+    if size.endswith("-l"):
+        f = "l"
+        size = size[:-2]
+    if size.endswith("-p"):
+        size = size[:-2]
+    rc = paper_sizes().get(size, (-1, -1))
+    if f == "p":
+        return rc
+    return (rc[1], rc[0])
+
+
+def paper_sizes():
+    """Known paper formats @ 72 dpi as a dictionary. Key is the format string
+    like "a4" for ISO-A4. Value is the tuple (width, height).
+
+    Information taken from the following web sites:
+    www.din-formate.de
+    www.din-formate.info/amerikanische-formate.html
+    www.directtools.de/wissen/normen/iso.htm
+    """
+    return {
+        "a0": (2384, 3370),
+        "a1": (1684, 2384),
+        "a10": (74, 105),
+        "a2": (1191, 1684),
+        "a3": (842, 1191),
+        "a4": (595, 842),
+        "a5": (420, 595),
+        "a6": (298, 420),
+        "a7": (210, 298),
+        "a8": (147, 210),
+        "a9": (105, 147),
+        "b0": (2835, 4008),
+        "b1": (2004, 2835),
+        "b10": (88, 125),
+        "b2": (1417, 2004),
+        "b3": (1001, 1417),
+        "b4": (709, 1001),
+        "b5": (499, 709),
+        "b6": (354, 499),
+        "b7": (249, 354),
+        "b8": (176, 249),
+        "b9": (125, 176),
+        "c0": (2599, 3677),
+        "c1": (1837, 2599),
+        "c10": (79, 113),
+        "c2": (1298, 1837),
+        "c3": (918, 1298),
+        "c4": (649, 918),
+        "c5": (459, 649),
+        "c6": (323, 459),
+        "c7": (230, 323),
+        "c8": (162, 230),
+        "c9": (113, 162),
+        "card-4x6": (288, 432),
+        "card-5x7": (360, 504),
+        "commercial": (297, 684),
+        "executive": (522, 756),
+        "invoice": (396, 612),
+        "ledger": (792, 1224),
+        "legal": (612, 1008),
+        "legal-13": (612, 936),
+        "letter": (612, 792),
+        "monarch": (279, 540),
+        "tabloid-extra": (864, 1296),
+        }
+
+def pdf_lookup_page_loc(doc, needle):
+    return mupdf.pdf_lookup_page_loc(doc, needle)
+
+
+def pdfobj_string(o, prefix=''):
+    '''
+    Returns description of mupdf.PdfObj (wrapper for pdf_obj) .
+    '''
+    assert 0, 'use mupdf.pdf_debug_obj() ?'
+    ret = ''
+    if mupdf.pdf_is_array(o):
+        l = mupdf.pdf_array_len(o)
+        ret += f'array {l}\n'
+        for i in range(l):
+            oo = mupdf.pdf_array_get(o, i)
+            ret += pdfobj_string(oo, prefix + '    ')
+            ret += '\n'
+    elif mupdf.pdf_is_bool(o):
+        ret += f'bool: {o.array_get_bool()}\n'
+    elif mupdf.pdf_is_dict(o):
+        l = mupdf.pdf_dict_len(o)
+        ret += f'dict {l}\n'
+        for i in range(l):
+            key = mupdf.pdf_dict_get_key(o, i)
+            value = mupdf.pdf_dict_get( o, key)
+            ret += f'{prefix} {key}: '
+            ret += pdfobj_string( value, prefix + '    ')
+            ret += '\n'
+    elif mupdf.pdf_is_embedded_file(o):
+        ret += f'embedded_file: {o.embedded_file_name()}\n'
+    elif mupdf.pdf_is_indirect(o):
+        ret += f'indirect: ...\n'
+    elif mupdf.pdf_is_int(o):
+        ret += f'int: {mupdf.pdf_to_int(o)}\n'
+    elif mupdf.pdf_is_jpx_image(o):
+        ret += f'jpx_image:\n'
+    elif mupdf.pdf_is_name(o):
+        ret += f'name: {mupdf.pdf_to_name(o)}\n'
+    elif o.pdf_is_null:
+        ret += f'null\n'
+    #elif o.pdf_is_number:
+    #    ret += f'number\n'
+    elif o.pdf_is_real:
+        ret += f'real: {o.pdf_to_real()}\n'
+    elif mupdf.pdf_is_stream(o):
+        ret += f'stream\n'
+    elif mupdf.pdf_is_string(o):
+        ret += f'string: {mupdf.pdf_to_string(o)}\n'
+    else:
+        ret += '<>\n'
+
+    return ret
+
+
+def repair_mono_font(page: "Page", font: "Font") -> None:
+    """Repair character spacing for mono fonts.
+
+    Notes:
+        Some mono-spaced fonts are displayed with a too large character
+        distance, e.g. "a b c" instead of "abc". This utility adds an entry
+        "/W[0 65535 w]" to the descendent font(s) of font. The float w is
+        taken to be the width of 0x20 (space).
+        This should enforce viewers to use 'w' as the character width.
+
+    Args:
+        page: pymupdf.Page object.
+        font: pymupdf.Font object.
+    """
+    if not font.flags["mono"]:  # font not flagged as monospaced
+        return None
+    doc = page.parent  # the document
+    fontlist = page.get_fonts()  # list of fonts on page
+    xrefs = [  # list of objects referring to font
+        f[0]
+        for f in fontlist
+        if (f[3] == font.name and f[4].startswith("F") and f[5].startswith("Identity"))
+    ]
+    if xrefs == []:  # our font does not occur
+        return
+    xrefs = set(xrefs)  # drop any double counts
+    width = int(round((font.glyph_advance(32) * 1000)))
+    for xref in xrefs:
+        if not TOOLS.set_font_width(doc, xref, width):
+            log("Cannot set width for '%s' in xref %i" % (font.name, xref))
+
+
+def sRGB_to_pdf(srgb: int) -> tuple:
+    """Convert sRGB color code to a PDF color triple.
+
+    There is **no error checking** for performance reasons!
+
+    Args:
+        srgb: (int) RRGGBB (red, green, blue), each color in range(255).
+    Returns:
+        Tuple (red, green, blue) each item in interval 0 <= item <= 1.
+    """
+    t = sRGB_to_rgb(srgb)
+    return t[0] / 255.0, t[1] / 255.0, t[2] / 255.0
+
+
+def sRGB_to_rgb(srgb: int) -> tuple:
+    """Convert sRGB color code to an RGB color triple.
+
+    There is **no error checking** for performance reasons!
+
+    Args:
+        srgb: (int) SSRRGGBB (red, green, blue), each color in range(255).
+        With MuPDF < 1.26, `s` is always 0.
+    Returns:
+        Tuple (red, green, blue) each item in interval 0 <= item <= 255.
+    """
+    srgb &= 0xffffff
+    r = srgb >> 16
+    g = (srgb - (r << 16)) >> 8
+    b = srgb - (r << 16) - (g << 8)
+    return (r, g, b)
+
+
+def string_in_names_list(p, names_list):
+    n = mupdf.pdf_array_len( names_list) if names_list else 0
+    str_ = mupdf.pdf_to_text_string( p)
+    for i in range(0, n, 2):
+        if mupdf.pdf_to_text_string( mupdf.pdf_array_get( names_list, i)) == str_:
+            return 1
+    return 0
+
+
+def strip_outline(doc, outlines, page_count, page_object_nums, names_list):
+    '''
+    Returns (count, first, prev).
+    '''
+    first = None
+    count = 0
+    current = outlines
+    prev = None
+    while current.m_internal:
+        # Strip any children to start with. This takes care of
+        # First / Last / Count for us.
+        nc = strip_outlines(doc, current, page_count, page_object_nums, names_list)
+
+        if not dest_is_valid(current, page_count, page_object_nums, names_list):
+            if nc == 0:
+                # Outline with invalid dest and no children. Drop it by
+                # pulling the next one in here.
+                next = mupdf.pdf_dict_get(current, PDF_NAME('Next'))
+                if not next.m_internal:
+                    # There is no next one to pull in
+                    if prev.m_internal:
+                        mupdf.pdf_dict_del(prev, PDF_NAME('Next'))
+                elif prev.m_internal:
+                    mupdf.pdf_dict_put(prev, PDF_NAME('Next'), next)
+                    mupdf.pdf_dict_put(next, PDF_NAME('Prev'), prev)
+                else:
+                    mupdf.pdf_dict_del(next, PDF_NAME('Prev'))
+                current = next
+            else:
+                # Outline with invalid dest, but children. Just drop the dest.
+                mupdf.pdf_dict_del(current, PDF_NAME('Dest'))
+                mupdf.pdf_dict_del(current, PDF_NAME('A'))
+                current = mupdf.pdf_dict_get(current, PDF_NAME('Next'))
+        else:
+            # Keep this one
+            if not first or not first.m_internal:
+                first = current
+            prev = current
+            current = mupdf.pdf_dict_get(current, PDF_NAME('Next'))
+            count += 1
+
+    return count, first, prev
+
+
+def strip_outlines(doc, outlines, page_count, page_object_nums, names_list):
+    if not outlines.m_internal:
+        return 0
+
+    first = mupdf.pdf_dict_get(outlines, PDF_NAME('First'))
+    if not first.m_internal:
+        nc = 0
+    else:
+        nc, first, last = strip_outline(doc, first, page_count, page_object_nums, names_list)
+
+    if nc == 0:
+        mupdf.pdf_dict_del(outlines, PDF_NAME('First'))
+        mupdf.pdf_dict_del(outlines, PDF_NAME('Last'))
+        mupdf.pdf_dict_del(outlines, PDF_NAME('Count'))
+    else:
+        old_count = mupdf.pdf_to_int(mupdf.pdf_dict_get(outlines, PDF_NAME('Count')))
+        mupdf.pdf_dict_put(outlines, PDF_NAME('First'), first)
+        mupdf.pdf_dict_put(outlines, PDF_NAME('Last'), last)
+        mupdf.pdf_dict_put(outlines, PDF_NAME('Count'), mupdf.pdf_new_int(nc if old_count > 0 else -nc))
+    return nc
+
+
+trace_device_FILL_PATH = 1
+trace_device_STROKE_PATH = 2
+trace_device_CLIP_PATH = 3
+trace_device_CLIP_STROKE_PATH = 4
+
+
+def unicode_to_glyph_name(ch: int) -> str:
+    """
+    Convenience function accessing unicodedata.
+    """
+    import unicodedata
+    try:
+        name = unicodedata.name(chr(ch))
+    except ValueError:
+        name = ".notdef"
+    return name
+
+
+def vdist(dir, a, b):
+    dx = b.x - a.x
+    dy = b.y - a.y
+    return mupdf.fz_abs(dx * dir.y + dy * dir.x)
+
+
+def apply_pages(
+        path,
+        pagefn,
+        *,
+        pagefn_args=(),
+        pagefn_kwargs=dict(),
+        initfn=None,
+        initfn_args=(),
+        initfn_kwargs=dict(),
+        pages=None,
+        method='single',
+        concurrency=None,
+        _stats=False,
+        ):
+    '''
+    Returns list of results from `pagefn()`, optionally using concurrency for
+    speed.
+    
+    Args:
+        path:
+            Path of document.
+        pagefn:
+            Function to call for each page; is passed (page, *pagefn_args,
+            **pagefn_kwargs). Return value is added to list that we return. If
+            `method` is not 'single', must be a top-level function - nested
+            functions don't work with concurrency.
+        pagefn_args
+        pagefn_kwargs:
+            Additional args to pass to `pagefn`. Must be picklable.
+        initfn:
+            If true, called once in each worker process; is passed
+            (*initfn_args, **initfn_kwargs).
+        initfn_args
+        initfn_kwargs:
+            Args to pass to initfn. Must be picklable.
+        pages:
+            List of page numbers to process, or None to include all pages.
+        method:
+            'single'
+                Do not use concurrency.
+            'mp'
+                Operate concurrently using Python's `multiprocessing` module.
+            'fork'
+                 Operate concurrently using custom implementation with
+                 `os.fork()`. Does not work on Windows.
+        concurrency:
+            Number of worker processes to use when operating concurrently. If
+            None, we use the number of available CPUs.
+        _stats:
+            Internal, may change or be removed. If true, we output simple
+            timing diagnostics.
+    
+    Note: We require a file path rather than a Document, because Document
+    instances do not work properly after a fork - internal file descriptor
+    offsets are shared between the parent and child processes.
+    '''
+    if _stats:
+        t0 = time.time()
+    
+    if method == 'single':
+        if initfn:
+            initfn(*initfn_args, **initfn_kwargs)
+        ret = list()
+        document = Document(path)
+        if pages is None:
+            pages = range(len(document))
+        for pno in pages:
+            page = document[pno]
+            r = pagefn(page, *pagefn_args, **initfn_kwargs)
+            ret.append(r)
+    
+    else:
+        # Use concurrency.
+        #
+        from . import _apply_pages
+    
+        if pages is None:
+            if _stats:
+                t = time.time()
+            with Document(path) as document:
+                num_pages = len(document)
+                pages = list(range(num_pages))
+            if _stats:
+                t = time.time() - t
+                log(f'{t:.2f}s: count pages.')
+    
+        if _stats:
+            t = time.time()
+        
+        if method == 'mp':
+            ret = _apply_pages._multiprocessing(
+                    path,
+                    pages,
+                    pagefn,
+                    pagefn_args,
+                    pagefn_kwargs,
+                    initfn,
+                    initfn_args,
+                    initfn_kwargs,
+                    concurrency,
+                    _stats,
+                    )
+    
+        elif method == 'fork':
+            ret = _apply_pages._fork(
+                    path,
+                    pages,
+                    pagefn,
+                    pagefn_args,
+                    pagefn_kwargs,
+                    initfn,
+                    initfn_args,
+                    initfn_kwargs,
+                    concurrency,
+                    _stats,
+                    )
+        
+        else:
+            assert 0, f'Unrecognised {method=}.'
+        
+        if _stats:
+            t = time.time() - t
+            log(f'{t:.2f}s: work.')
+
+    if _stats:
+        t = time.time() - t0
+        log(f'{t:.2f}s: total.')
+    return ret
+
+
+def get_text(
+        path,
+        *,
+        pages=None,
+        method='single',
+        concurrency=None,
+        
+        option='text',
+        clip=None,
+        flags=None,
+        textpage=None,
+        sort=False,
+        delimiters=None,
+        
+        _stats=False,
+        ):
+    '''
+    Returns list of results from `Page.get_text()`, optionally using
+    concurrency for speed.
+    
+    Args:
+        path:
+            Path of document.
+        pages:
+            List of page numbers to process, or None to include all pages.
+        method:
+            'single'
+                Do not use concurrency.
+            'mp'
+                Operate concurrently using Python's `multiprocessing` module.
+            'fork'
+                 Operate concurrently using custom implementation with
+                 `os.fork`. Does not work on Windows.
+        concurrency:
+            Number of worker processes to use when operating concurrently. If
+            None, we use the number of available CPUs.
+        option
+        clip
+        flags
+        textpage
+        sort
+        delimiters:
+            Passed to internal calls to `Page.get_text()`.
+    '''
+    args_dict = dict(
+            option=option,
+            clip=clip,
+            flags=flags,
+            textpage=textpage,
+            sort=sort,
+            delimiters=delimiters,
+            )
+    
+    return apply_pages(
+            path,
+            Page.get_text,
+            pagefn_kwargs=args_dict,
+            pages=pages,
+            method=method,
+            concurrency=concurrency,
+            _stats=_stats,
+            )
+
+
+class TOOLS:
+    '''
+    We use @staticmethod to avoid the need to create an instance of this class.
+    '''
+
+    def _derotate_matrix(page):
+        if isinstance(page, mupdf.PdfPage):
+            return JM_py_from_matrix(JM_derotate_page_matrix(page))
+        else:
+            return JM_py_from_matrix(mupdf.FzMatrix())
+
+    @staticmethod
+    def _fill_widget(annot, widget):
+        val = JM_get_widget_properties(annot, widget)
+
+        widget.rect = Rect(annot.rect)
+        widget.xref = annot.xref
+        widget.parent = annot.parent
+        widget._annot = annot  # backpointer to annot object
+        if not widget.script:
+            widget.script = None
+        if not widget.script_stroke:
+            widget.script_stroke = None
+        if not widget.script_format:
+            widget.script_format = None
+        if not widget.script_change:
+            widget.script_change = None
+        if not widget.script_calc:
+            widget.script_calc = None
+        if not widget.script_blur:
+            widget.script_blur = None
+        if not widget.script_focus:
+            widget.script_focus = None
+        return val
+
+    @staticmethod
+    def _get_all_contents(page):
+        page = _as_pdf_page(page.this)
+        res = JM_read_contents(page.obj())
+        result = JM_BinFromBuffer( res)
+        return result
+
+    @staticmethod
+    def _insert_contents(page, newcont, overlay=1):
+        """Add bytes as a new /Contents object for a page, and return its xref."""
+        pdfpage = _as_pdf_page(page, required=1)
+        contbuf = JM_BufferFromBytes(newcont)
+        xref = JM_insert_contents(pdfpage.doc(), pdfpage.obj(), contbuf, overlay)
+        #fixme: pdfpage->doc->dirty = 1;
+        return xref
+
+    @staticmethod
+    def _le_annot_parms(annot, p1, p2, fill_color):
+        """Get common parameters for making annot line end symbols.
+
+        Returns:
+            m: matrix that maps p1, p2 to points L, P on the x-axis
+            im: its inverse
+            L, P: transformed p1, p2
+            w: line width
+            scol: stroke color string
+            fcol: fill color store_shrink
+            opacity: opacity string (gs command)
+        """
+        w = annot.border["width"]  # line width
+        sc = annot.colors["stroke"]  # stroke color
+        if not sc:  # black if missing
+            sc = (0,0,0)
+        scol = " ".join(map(str, sc)) + " RG\n"
+        if fill_color:
+            fc = fill_color
+        else:
+            fc = annot.colors["fill"]  # fill color
+        if not fc:
+            fc = (1,1,1)  # white if missing
+        fcol = " ".join(map(str, fc)) + " rg\n"
+        # nr = annot.rect
+        np1 = p1                   # point coord relative to annot rect
+        np2 = p2                   # point coord relative to annot rect
+        m = Matrix(util_hor_matrix(np1, np2))  # matrix makes the line horizontal
+        im = ~m                            # inverted matrix
+        L = np1 * m                        # converted start (left) point
+        R = np2 * m                        # converted end (right) point
+        if 0 <= annot.opacity < 1:
+            opacity = "/H gs\n"
+        else:
+            opacity = ""
+        return m, im, L, R, w, scol, fcol, opacity
+
+    @staticmethod
+    def _le_butt(annot, p1, p2, lr, fill_color):
+        """Make stream commands for butt line end symbol. "lr" denotes left (False) or right point.
+        """
+        m, im, L, R, w, scol, fcol, opacity = TOOLS._le_annot_parms(annot, p1, p2, fill_color)
+        shift = 3
+        d = shift * max(1, w)
+        M = R if lr else L
+        top = (M + (0, -d/2.)) * im
+        bot = (M + (0, d/2.)) * im
+        ap = "\nq\n%s%f %f m\n" % (opacity, top.x, top.y)
+        ap += "%f %f l\n" % (bot.x, bot.y)
+        ap += _format_g(w) + " w\n"
+        ap += scol + "s\nQ\n"
+        return ap
+
+    @staticmethod
+    def _le_circle(annot, p1, p2, lr, fill_color):
+        """Make stream commands for circle line end symbol. "lr" denotes left (False) or right point.
+        """
+        m, im, L, R, w, scol, fcol, opacity = TOOLS._le_annot_parms(annot, p1, p2, fill_color)
+        shift = 2.5             # 2*shift*width = length of square edge
+        d = shift * max(1, w)
+        M = R - (d/2., 0) if lr else L + (d/2., 0)
+        r = Rect(M, M) + (-d, -d, d, d)         # the square
+        ap = "q\n" + opacity + TOOLS._oval_string(r.tl * im, r.tr * im, r.br * im, r.bl * im)
+        ap += _format_g(w) + " w\n"
+        ap += scol + fcol + "b\nQ\n"
+        return ap
+
+    @staticmethod
+    def _le_closedarrow(annot, p1, p2, lr, fill_color):
+        """Make stream commands for closed arrow line end symbol. "lr" denotes left (False) or right point.
+        """
+        m, im, L, R, w, scol, fcol, opacity = TOOLS._le_annot_parms(annot, p1, p2, fill_color)
+        shift = 2.5
+        d = shift * max(1, w)
+        p2 = R + (d/2., 0) if lr else L - (d/2., 0)
+        p1 = p2 + (-2*d, -d) if lr else p2 + (2*d, -d)
+        p3 = p2 + (-2*d, d) if lr else p2 + (2*d, d)
+        p1 *= im
+        p2 *= im
+        p3 *= im
+        ap = "\nq\n%s%f %f m\n" % (opacity, p1.x, p1.y)
+        ap += "%f %f l\n" % (p2.x, p2.y)
+        ap += "%f %f l\n" % (p3.x, p3.y)
+        ap += _format_g(w) + " w\n"
+        ap += scol + fcol + "b\nQ\n"
+        return ap
+
+    @staticmethod
+    def _le_diamond(annot, p1, p2, lr, fill_color):
+        """Make stream commands for diamond line end symbol. "lr" denotes left (False) or right point.
+        """
+        m, im, L, R, w, scol, fcol, opacity = TOOLS._le_annot_parms(annot, p1, p2, fill_color)
+        shift = 2.5             # 2*shift*width = length of square edge
+        d = shift * max(1, w)
+        M = R - (d/2., 0) if lr else L + (d/2., 0)
+        r = Rect(M, M) + (-d, -d, d, d)         # the square
+        # the square makes line longer by (2*shift - 1)*width
+        p = (r.tl + (r.bl - r.tl) * 0.5) * im
+        ap = "q\n%s%f %f m\n" % (opacity, p.x, p.y)
+        p = (r.tl + (r.tr - r.tl) * 0.5) * im
+        ap += "%f %f l\n"   % (p.x, p.y)
+        p = (r.tr + (r.br - r.tr) * 0.5) * im
+        ap += "%f %f l\n"   % (p.x, p.y)
+        p = (r.br + (r.bl - r.br) * 0.5) * im
+        ap += "%f %f l\n"   % (p.x, p.y)
+        ap += _format_g(w) + " w\n"
+        ap += scol + fcol + "b\nQ\n"
+        return ap
+
+    @staticmethod
+    def _le_openarrow(annot, p1, p2, lr, fill_color):
+        """Make stream commands for open arrow line end symbol. "lr" denotes left (False) or right point.
+        """
+        m, im, L, R, w, scol, fcol, opacity = TOOLS._le_annot_parms(annot, p1, p2, fill_color)
+        shift = 2.5
+        d = shift * max(1, w)
+        p2 = R + (d/2., 0) if lr else L - (d/2., 0)
+        p1 = p2 + (-2*d, -d) if lr else p2 + (2*d, -d)
+        p3 = p2 + (-2*d, d) if lr else p2 + (2*d, d)
+        p1 *= im
+        p2 *= im
+        p3 *= im
+        ap = "\nq\n%s%f %f m\n" % (opacity, p1.x, p1.y)
+        ap += "%f %f l\n" % (p2.x, p2.y)
+        ap += "%f %f l\n" % (p3.x, p3.y)
+        ap += _format_g(w) + " w\n"
+        ap += scol + "S\nQ\n"
+        return ap
+
+    @staticmethod
+    def _le_rclosedarrow(annot, p1, p2, lr, fill_color):
+        """Make stream commands for right closed arrow line end symbol. "lr" denotes left (False) or right point.
+        """
+        m, im, L, R, w, scol, fcol, opacity = TOOLS._le_annot_parms(annot, p1, p2, fill_color)
+        shift = 2.5
+        d = shift * max(1, w)
+        p2 = R - (2*d, 0) if lr else L + (2*d, 0)
+        p1 = p2 + (2*d, -d) if lr else p2 + (-2*d, -d)
+        p3 = p2 + (2*d, d) if lr else p2 + (-2*d, d)
+        p1 *= im
+        p2 *= im
+        p3 *= im
+        ap = "\nq\n%s%f %f m\n" % (opacity, p1.x, p1.y)
+        ap += "%f %f l\n" % (p2.x, p2.y)
+        ap += "%f %f l\n" % (p3.x, p3.y)
+        ap += _format_g(w) + " w\n"
+        ap += scol + fcol + "b\nQ\n"
+        return ap
+
+    @staticmethod
+    def _le_ropenarrow(annot, p1, p2, lr, fill_color):
+        """Make stream commands for right open arrow line end symbol. "lr" denotes left (False) or right point.
+        """
+        m, im, L, R, w, scol, fcol, opacity = TOOLS._le_annot_parms(annot, p1, p2, fill_color)
+        shift = 2.5
+        d = shift * max(1, w)
+        p2 = R - (d/3., 0) if lr else L + (d/3., 0)
+        p1 = p2 + (2*d, -d) if lr else p2 + (-2*d, -d)
+        p3 = p2 + (2*d, d) if lr else p2 + (-2*d, d)
+        p1 *= im
+        p2 *= im
+        p3 *= im
+        ap = "\nq\n%s%f %f m\n" % (opacity, p1.x, p1.y)
+        ap += "%f %f l\n" % (p2.x, p2.y)
+        ap += "%f %f l\n" % (p3.x, p3.y)
+        ap += _format_g(w) + " w\n"
+        ap += scol + fcol + "S\nQ\n"
+        return ap
+
+    @staticmethod
+    def _le_slash(annot, p1, p2, lr, fill_color):
+        """Make stream commands for slash line end symbol. "lr" denotes left (False) or right point.
+        """
+        m, im, L, R, w, scol, fcol, opacity = TOOLS._le_annot_parms(annot, p1, p2, fill_color)
+        rw = 1.1547 * max(1, w) * 1.0         # makes rect diagonal a 30 deg inclination
+        M = R if lr else L
+        r = Rect(M.x - rw, M.y - 2 * w, M.x + rw, M.y + 2 * w)
+        top = r.tl * im
+        bot = r.br * im
+        ap = "\nq\n%s%f %f m\n" % (opacity, top.x, top.y)
+        ap += "%f %f l\n" % (bot.x, bot.y)
+        ap += _format_g(w) + " w\n"
+        ap += scol + "s\nQ\n"
+        return ap
+
+    @staticmethod
+    def _le_square(annot, p1, p2, lr, fill_color):
+        """Make stream commands for square line end symbol. "lr" denotes left (False) or right point.
+        """
+        m, im, L, R, w, scol, fcol, opacity = TOOLS._le_annot_parms(annot, p1, p2, fill_color)
+        shift = 2.5             # 2*shift*width = length of square edge
+        d = shift * max(1, w)
+        M = R - (d/2., 0) if lr else L + (d/2., 0)
+        r = Rect(M, M) + (-d, -d, d, d)         # the square
+        # the square makes line longer by (2*shift - 1)*width
+        p = r.tl * im
+        ap = "q\n%s%f %f m\n" % (opacity, p.x, p.y)
+        p = r.tr * im
+        ap += "%f %f l\n"   % (p.x, p.y)
+        p = r.br * im
+        ap += "%f %f l\n"   % (p.x, p.y)
+        p = r.bl * im
+        ap += "%f %f l\n"   % (p.x, p.y)
+        ap += _format_g(w) + " w\n"
+        ap += scol + fcol + "b\nQ\n"
+        return ap
+
+    @staticmethod
+    def _oval_string(p1, p2, p3, p4):
+        """Return /AP string defining an oval within a 4-polygon provided as points
+        """
+        def bezier(p, q, r):
+            f = "%f %f %f %f %f %f c\n"
+            return f % (p.x, p.y, q.x, q.y, r.x, r.y)
+
+        kappa = 0.55228474983              # magic number
+        ml = p1 + (p4 - p1) * 0.5          # middle points ...
+        mo = p1 + (p2 - p1) * 0.5          # for each ...
+        mr = p2 + (p3 - p2) * 0.5          # polygon ...
+        mu = p4 + (p3 - p4) * 0.5          # side
+        ol1 = ml + (p1 - ml) * kappa       # the 8 bezier
+        ol2 = mo + (p1 - mo) * kappa       # helper points
+        or1 = mo + (p2 - mo) * kappa
+        or2 = mr + (p2 - mr) * kappa
+        ur1 = mr + (p3 - mr) * kappa
+        ur2 = mu + (p3 - mu) * kappa
+        ul1 = mu + (p4 - mu) * kappa
+        ul2 = ml + (p4 - ml) * kappa
+        # now draw, starting from middle point of left side
+        ap = "%f %f m\n" % (ml.x, ml.y)
+        ap += bezier(ol1, ol2, mo)
+        ap += bezier(or1, or2, mr)
+        ap += bezier(ur1, ur2, mu)
+        ap += bezier(ul1, ul2, ml)
+        return ap
+
+    @staticmethod
+    def _parse_da(annot):
+
+        if g_use_extra:
+            val = extra.Tools_parse_da( annot.this)
+        else:
+            def Tools__parse_da(annot):
+                this_annot = annot.this
+                assert isinstance(this_annot, mupdf.PdfAnnot)
+                this_annot_obj = mupdf.pdf_annot_obj( this_annot)
+                pdf = mupdf.pdf_get_bound_document( this_annot_obj)
+                try:
+                    da = mupdf.pdf_dict_get_inheritable( this_annot_obj, PDF_NAME('DA'))
+                    if not da.m_internal:
+                        trailer = mupdf.pdf_trailer(pdf)
+                        da = mupdf.pdf_dict_getl(trailer,
+                                PDF_NAME('Root'),
+                                PDF_NAME('AcroForm'),
+                                PDF_NAME('DA'),
+                                )
+                    da_str = mupdf.pdf_to_text_string(da)
+                except Exception:
+                    if g_exceptions_verbose:    exception_info()
+                    return
+                return da_str
+            val = Tools__parse_da(annot)
+
+        if not val:
+            return ((0,), "", 0)
+        font = "Helv"
+        fsize = 12
+        col = (0, 0, 0)
+        dat = val.split()  # split on any whitespace
+        for i, item in enumerate(dat):
+            if item == "Tf":
+                font = dat[i - 2][1:]
+                fsize = float(dat[i - 1])
+                dat[i] = dat[i-1] = dat[i-2] = ""
+                continue
+            if item == "g":            # unicolor text
+                col = [(float(dat[i - 1]))]
+                dat[i] = dat[i-1] = ""
+                continue
+            if item == "rg":           # RGB colored text
+                col = [float(f) for f in dat[i - 3:i]]
+                dat[i] = dat[i-1] = dat[i-2] = dat[i-3] = ""
+                continue
+            if item == "k":           # CMYK colored text
+                col = [float(f) for f in dat[i - 4:i]]
+                dat[i] = dat[i-1] = dat[i-2] = dat[i-3] = dat[i-4] = ""
+                continue
+
+        val = (col, font, fsize)
+        return val
+
+    @staticmethod
+    def _reset_widget(annot):
+        this_annot = annot
+        this_annot_obj = mupdf.pdf_annot_obj(this_annot)
+        pdf = mupdf.pdf_get_bound_document(this_annot_obj)
+        mupdf.pdf_field_reset(pdf, this_annot_obj)
+
+    @staticmethod
+    def _rotate_matrix(page):
+        pdfpage = page._pdf_page(required=False)
+        if not pdfpage.m_internal:
+            return JM_py_from_matrix(mupdf.FzMatrix())
+        return JM_py_from_matrix(JM_rotate_page_matrix(pdfpage))
+
+    @staticmethod
+    def _save_widget(annot, widget):
+        JM_set_widget_properties(annot, widget)
+
+    def _update_da(annot, da_str):
+        if g_use_extra:
+            extra.Tools_update_da( annot.this, da_str)
+        else:
+            try:
+                this_annot = annot.this
+                assert isinstance(this_annot, mupdf.PdfAnnot)
+                mupdf.pdf_dict_put_text_string(mupdf.pdf_annot_obj(this_annot), PDF_NAME('DA'), da_str)
+                mupdf.pdf_dict_del(mupdf.pdf_annot_obj(this_annot), PDF_NAME('DS'))    # /* not supported */
+                mupdf.pdf_dict_del(mupdf.pdf_annot_obj(this_annot), PDF_NAME('RC'))    # /* not supported */
+            except Exception:
+                if g_exceptions_verbose:    exception_info()
+                return
+            return
+    
+    @staticmethod
+    def gen_id():
+        global TOOLS_JM_UNIQUE_ID
+        TOOLS_JM_UNIQUE_ID += 1
+        return TOOLS_JM_UNIQUE_ID
+
+    @staticmethod
+    def glyph_cache_empty():
+        '''
+        Empty the glyph cache.
+        '''
+        mupdf.fz_purge_glyph_cache()
+
+    @staticmethod
+    def image_profile(stream, keep_image=0):
+        '''
+        Metadata of an image binary stream.
+        '''
+        return JM_image_profile(stream, keep_image)
+    
+    @staticmethod
+    def mupdf_display_errors(on=None):
+        '''
+        Set MuPDF error display to True or False.
+        '''
+        global JM_mupdf_show_errors
+        if on is not None:
+            JM_mupdf_show_errors = bool(on)
+        return JM_mupdf_show_errors
+
+    @staticmethod
+    def mupdf_display_warnings(on=None):
+        '''
+        Set MuPDF warnings display to True or False.
+        '''
+        global JM_mupdf_show_warnings
+        if on is not None:
+            JM_mupdf_show_warnings = bool(on)
+        return JM_mupdf_show_warnings
+
+    @staticmethod
+    def mupdf_version():
+        '''Get version of MuPDF binary build.'''
+        return mupdf.FZ_VERSION
+
+    @staticmethod
+    def mupdf_warnings(reset=1):
+        '''
+        Get the MuPDF warnings/errors with optional reset (default).
+        '''
+        # Get any trailing `... repeated  times...` message.
+        mupdf.fz_flush_warnings()
+        ret = '\n'.join( JM_mupdf_warnings_store)
+        if reset:
+            TOOLS.reset_mupdf_warnings()
+        return ret
+
+    @staticmethod
+    def reset_mupdf_warnings():
+        global JM_mupdf_warnings_store
+        JM_mupdf_warnings_store = list()
+        
+    @staticmethod
+    def set_aa_level(level):
+        '''
+        Set anti-aliasing level.
+        '''
+        mupdf.fz_set_aa_level(level)
+    
+    @staticmethod
+    def set_annot_stem( stem=None):
+        global JM_annot_id_stem
+        if stem is None:
+            return JM_annot_id_stem
+        len_ = len(stem) + 1
+        if len_ > 50:
+            len_ = 50
+        JM_annot_id_stem = stem[:50]
+        return JM_annot_id_stem
+
+    @staticmethod
+    def set_font_width(doc, xref, width):
+        pdf = _as_pdf_document(doc, required=0)
+        if not pdf.m_internal:
+            return False
+        font = mupdf.pdf_load_object(pdf, xref)
+        dfonts = mupdf.pdf_dict_get(font, PDF_NAME('DescendantFonts'))
+        if mupdf.pdf_is_array(dfonts):
+            n = mupdf.pdf_array_len(dfonts)
+            for i in range(n):
+                dfont = mupdf.pdf_array_get(dfonts, i)
+                warray = mupdf.pdf_new_array(pdf, 3)
+                mupdf.pdf_array_push(warray, mupdf.pdf_new_int(0))
+                mupdf.pdf_array_push(warray, mupdf.pdf_new_int(65535))
+                mupdf.pdf_array_push(warray, mupdf.pdf_new_int(width))
+                mupdf.pdf_dict_put(dfont, PDF_NAME('W'), warray)
+        return True
+
+    @staticmethod
+    def set_graphics_min_line_width(min_line_width):
+        '''
+        Set the graphics minimum line width.
+        '''
+        mupdf.fz_set_graphics_min_line_width(min_line_width)
+
+    @staticmethod
+    def set_icc( on=0):
+        """Set ICC color handling on or off."""
+        if on:
+            if mupdf.FZ_ENABLE_ICC:
+                mupdf.fz_enable_icc()
+            else:
+                RAISEPY( "MuPDF built w/o ICC support",PyExc_ValueError)
+        elif mupdf.FZ_ENABLE_ICC:
+            mupdf.fz_disable_icc()
+ 
+    @staticmethod
+    def set_low_memory( on=None):
+        """Set / unset MuPDF device caching."""
+        if on is not None:
+            _globals.no_device_caching = bool(on)
+        return _globals.no_device_caching
+
+    @staticmethod
+    def set_small_glyph_heights(on=None):
+        """Set / unset small glyph heights."""
+        if on is not None:
+            _globals.small_glyph_heights = bool(on)
+            if g_use_extra:
+                extra.set_small_glyph_heights(_globals.small_glyph_heights)
+        return _globals.small_glyph_heights
+    
+    @staticmethod
+    def set_subset_fontnames(on=None):
+        '''
+        Set / unset returning fontnames with their subset prefix.
+        '''
+        if on is not None:
+            _globals.subset_fontnames = bool(on)
+            if g_use_extra:
+                extra.set_subset_fontnames(_globals.subset_fontnames)
+        return _globals.subset_fontnames
+    
+    @staticmethod
+    def show_aa_level():
+        '''
+        Show anti-aliasing values.
+        '''
+        return dict(
+                graphics = mupdf.fz_graphics_aa_level(),
+                text = mupdf.fz_text_aa_level(),
+                graphics_min_line_width = mupdf.fz_graphics_min_line_width(),
+                )
+
+    @staticmethod
+    def store_maxsize():
+        '''
+        MuPDF store size limit.
+        '''
+        # fixme: return gctx->store->max.
+        return None
+
+    @staticmethod
+    def store_shrink(percent):
+        '''
+        Free 'percent' of current store size.
+        '''
+        if percent >= 100:
+            mupdf.fz_empty_store()
+            return 0
+        if percent > 0:
+            mupdf.fz_shrink_store( 100 - percent)
+        # fixme: return gctx->store->size.
+    
+    @staticmethod
+    def store_size():
+        '''
+        MuPDF current store size.
+        '''
+        # fixme: return gctx->store->size.
+        return None
+    
+    @staticmethod
+    def unset_quad_corrections(on=None):
+        '''
+        Set ascender / descender corrections on or off.
+        '''
+        if on is not None:
+            _globals.skip_quad_corrections = bool(on)
+            if g_use_extra:
+                extra.set_skip_quad_corrections(_globals.skip_quad_corrections)
+        return _globals.skip_quad_corrections
+
+    # fixme: also defined at top-level.
+    JM_annot_id_stem = 'fitz'
+
+    fitz_config = JM_fitz_config()
+
+
+# Callbacks not yet supported with cppyy.
+if not mupdf_cppyy:
+    mupdf.fz_set_warning_callback(JM_mupdf_warning)
+    mupdf.fz_set_error_callback(JM_mupdf_error)
+
+
+# If there are pending warnings when we exit, we end up in this sequence:
+#
+#   atexit()
+#   -> mupdf::internal_thread_state::~internal_thread_state()
+#   -> fz_drop_context()
+#   -> fz_flush_warnings()
+#   -> SWIG Director code
+#   -> Python calling JM_mupdf_warning().
+#
+# Unfortunately this causes a SEGV, seemingly because the SWIG Director code has
+# already been torn down.
+#
+# So we use a Python atexit handler to explicitly call fz_flush_warnings();
+# this appears to happen early enough for the Director machinery to still
+# work. So in the sequence above, fz_flush_warnings() will find that there are
+# no pending warnings and will not attempt to call JM_mupdf_warning().
+#
+def _atexit():
+    #log( 'PyMuPDF/src/__init__.py:_atexit() called')
+    mupdf.fz_flush_warnings()
+    mupdf.fz_set_warning_callback(None)
+    mupdf.fz_set_error_callback(None)
+    #log( '_atexit() returning')
+atexit.register( _atexit)
+
+
+# List of (name, red, green, blue) where:
+#   name: upper-case name.
+#   red, green, blue: integer in range 0..255.
+#
+from . import _wxcolors
+_wxcolors = _wxcolors._wxcolors
+
+
+# Dict mapping from name to (red, green, blue).
+#   name: lower-case name.
+#   red, green, blue: float in range 0..1.
+#
+pdfcolor = dict()
+for name, r, g, b in _wxcolors:
+    pdfcolor[name.lower()] = (r/255, g/255, b/255)
+
+
+def colors_pdf_dict():
+    '''
+    Returns dict mapping from name to (red, green, blue).
+        name: lower-case name.
+        red, green, blue: float in range 0..1.
+    '''
+    return pdfcolor
+
+
+def colors_wx_list():
+    '''
+    Returns list of (name, red, green, blue) tuples:
+        name: upper-case name.
+        red, green, blue: integers in range 0..255.
+    '''
+    return _wxcolors
+
+
+# We cannot import utils earlier because it imports this .py file itself and
+# uses some pymupdf.* types in function typing.
+#
+from . import utils
+
+
+# Use utils.*() fns for some class methods.
+#
+recover_bbox_quad           = utils.recover_bbox_quad
+recover_char_quad           = utils.recover_char_quad
+recover_line_quad           = utils.recover_line_quad
+recover_quad                = utils.recover_quad
+recover_span_quad           = utils.recover_span_quad
+
+Annot.get_text              = utils.get_text
+Annot.get_textbox           = utils.get_textbox
+
+Document._do_links          = utils.do_links
+Document._do_widgets        = utils.do_widgets
+Document.del_toc_item       = utils.del_toc_item
+Document.get_char_widths    = utils.get_char_widths
+Document.get_oc             = utils.get_oc
+Document.get_ocmd           = utils.get_ocmd
+Document.get_page_labels    = utils.get_page_labels
+Document.get_page_numbers   = utils.get_page_numbers
+Document.get_page_pixmap    = utils.get_page_pixmap
+Document.get_page_text      = utils.get_page_text
+Document.get_toc            = utils.get_toc
+Document.has_annots         = utils.has_annots
+Document.has_links          = utils.has_links
+Document.insert_page        = utils.insert_page
+Document.new_page           = utils.new_page
+Document.scrub              = utils.scrub
+Document.search_page_for    = utils.search_page_for
+Document.set_metadata       = utils.set_metadata
+Document.set_oc             = utils.set_oc
+Document.set_ocmd           = utils.set_ocmd
+Document.set_page_labels    = utils.set_page_labels
+Document.set_toc            = utils.set_toc
+Document.set_toc_item       = utils.set_toc_item
+Document.subset_fonts       = utils.subset_fonts
+Document.tobytes            = Document.write
+Document.xref_copy          = utils.xref_copy
+
+IRect.get_area              = utils.get_area
+
+Page.apply_redactions       = utils.apply_redactions
+Page.delete_image           = utils.delete_image
+Page.delete_widget          = utils.delete_widget
+Page.draw_bezier            = utils.draw_bezier
+Page.draw_circle            = utils.draw_circle
+Page.draw_curve             = utils.draw_curve
+Page.draw_line              = utils.draw_line
+Page.draw_oval              = utils.draw_oval
+Page.draw_polyline          = utils.draw_polyline
+Page.draw_quad              = utils.draw_quad
+Page.draw_rect              = utils.draw_rect
+Page.draw_sector            = utils.draw_sector
+Page.draw_squiggle          = utils.draw_squiggle
+Page.draw_zigzag            = utils.draw_zigzag
+Page.get_image_info         = utils.get_image_info
+Page.get_image_rects        = utils.get_image_rects
+Page.get_label              = utils.get_label
+Page.get_links              = utils.get_links
+Page.get_pixmap             = utils.get_pixmap
+Page.get_text               = utils.get_text
+Page.get_text_blocks        = utils.get_text_blocks
+Page.get_text_selection     = utils.get_text_selection
+Page.get_text_words         = utils.get_text_words
+Page.get_textbox            = utils.get_textbox
+Page.get_textpage_ocr       = utils.get_textpage_ocr
+Page.insert_image           = utils.insert_image
+Page.insert_link            = utils.insert_link
+Page.insert_text            = utils.insert_text
+Page.insert_textbox         = utils.insert_textbox
+Page.insert_htmlbox         = utils.insert_htmlbox
+Page.new_shape              = lambda x: utils.Shape(x)
+Page.replace_image          = utils.replace_image
+Page.search_for             = utils.search_for
+Page.show_pdf_page          = utils.show_pdf_page
+Page.update_link            = utils.update_link
+Page.write_text             = utils.write_text
+from .table import find_tables
+
+Page.find_tables = find_tables
+
+Rect.get_area               = utils.get_area
+
+TextWriter.fill_textbox     = utils.fill_textbox
+
+
+class FitzDeprecation(DeprecationWarning):
+    pass
+
+def restore_aliases():
+    warnings.filterwarnings( "once", category=FitzDeprecation)
+
+    def showthis(msg, cat, filename, lineno, file=None, line=None):
+        text = warnings.formatwarning(msg, cat, filename, lineno, line=line)
+        s = text.find("FitzDeprecation")
+        if s < 0:
+            log(text)
+            return
+        text = text[s:].splitlines()[0][4:]
+        log(text)
+
+    warnings.showwarning = showthis
+
+    def _alias(class_, new_name, legacy_name=None):
+        '''
+        Adds an alias for a class_ or module item clled ..
+
+        class_:
+            Class/module to modify; use None for the current module.
+        new_name:
+            String name of existing item, e.g. name of method.
+        legacy_name:
+            Name of legacy object to create in . If None, we generate
+            from  by removing underscores and capitalising the next
+            letter.
+        '''
+        if class_ is None:
+            class_ = sys.modules[__name__]
+        if not legacy_name:
+            legacy_name = ''
+            capitalise_next = False
+            for c in new_name:
+                if c == '_':
+                    capitalise_next = True
+                elif capitalise_next:
+                    legacy_name += c.upper()
+                    capitalise_next = False
+                else:
+                    legacy_name += c
+        new_object = getattr( class_, new_name)
+        assert not getattr( class_, legacy_name, None), f'class {class_} already has {legacy_name}'
+        if callable( new_object):
+            def deprecated_function( *args, **kwargs):
+                warnings.warn(
+                        f'"{legacy_name=}" removed from {class_} after v1.19.0 - use "{new_name}".',
+                        category=FitzDeprecation,
+                        )
+                return new_object( *args, **kwargs)
+            setattr( class_, legacy_name, deprecated_function)
+            deprecated_function.__doc__ = (
+                    f'*** Deprecated and removed in version after v1.19.0 - use "{new_name}". ***\n'
+                    f'{new_object.__doc__}'
+                    )
+        else:
+            setattr( class_, legacy_name, new_object)
+
+    _alias( Annot, 'get_file',              'fileGet')
+    _alias( Annot, 'get_pixmap')
+    _alias( Annot, 'get_sound',             'soundGet')
+    _alias( Annot, 'get_text')
+    _alias( Annot, 'get_textbox')
+    _alias( Annot, 'get_textpage',          'getTextPage')
+    _alias( Annot, 'line_ends')
+    _alias( Annot, 'set_blendmode',         'setBlendMode')
+    _alias( Annot, 'set_border')
+    _alias( Annot, 'set_colors')
+    _alias( Annot, 'set_flags')
+    _alias( Annot, 'set_info')
+    _alias( Annot, 'set_line_ends')
+    _alias( Annot, 'set_name')
+    _alias( Annot, 'set_oc', 'setOC')
+    _alias( Annot, 'set_opacity')
+    _alias( Annot, 'set_rect')
+    _alias( Annot, 'update_file',           'fileUpd')
+    _alias( DisplayList, 'get_pixmap')
+    _alias( DisplayList, 'get_textpage',    'getTextPage')
+    _alias( Document, 'chapter_count')
+    _alias( Document, 'chapter_page_count')
+    _alias( Document, 'convert_to_pdf',     'convertToPDF')
+    _alias( Document, 'copy_page')
+    _alias( Document, 'delete_page')
+    _alias( Document, 'delete_pages',       'deletePageRange')
+    _alias( Document, 'embfile_add',        'embeddedFileAdd')
+    _alias( Document, 'embfile_count',      'embeddedFileCount')
+    _alias( Document, 'embfile_del',        'embeddedFileDel')
+    _alias( Document, 'embfile_get',        'embeddedFileGet')
+    _alias( Document, 'embfile_info',       'embeddedFileInfo')
+    _alias( Document, 'embfile_names',      'embeddedFileNames')
+    _alias( Document, 'embfile_upd',        'embeddedFileUpd')
+    _alias( Document, 'extract_font')
+    _alias( Document, 'extract_image')
+    _alias( Document, 'find_bookmark')
+    _alias( Document, 'fullcopy_page')
+    _alias( Document, 'get_char_widths')
+    _alias( Document, 'get_ocgs',           'getOCGs')
+    _alias( Document, 'get_page_fonts',     'getPageFontList')
+    _alias( Document, 'get_page_images',    'getPageImageList')
+    _alias( Document, 'get_page_pixmap')
+    _alias( Document, 'get_page_text')
+    _alias( Document, 'get_page_xobjects',  'getPageXObjectList')
+    _alias( Document, 'get_sigflags',       'getSigFlags')
+    _alias( Document, 'get_toc', 'getToC')
+    _alias( Document, 'get_xml_metadata')
+    _alias( Document, 'insert_page')
+    _alias( Document, 'insert_pdf',         'insertPDF')
+    _alias( Document, 'is_dirty')
+    _alias( Document, 'is_form_pdf',        'isFormPDF')
+    _alias( Document, 'is_pdf', 'isPDF')
+    _alias( Document, 'is_reflowable')
+    _alias( Document, 'is_repaired')
+    _alias( Document, 'last_location')
+    _alias( Document, 'load_page')
+    _alias( Document, 'make_bookmark')
+    _alias( Document, 'move_page')
+    _alias( Document, 'needs_pass')
+    _alias( Document, 'new_page')
+    _alias( Document, 'next_location')
+    _alias( Document, 'page_count')
+    _alias( Document, 'page_cropbox',       'pageCropBox')
+    _alias( Document, 'page_xref')
+    _alias( Document, 'pdf_catalog',        'PDFCatalog')
+    _alias( Document, 'pdf_trailer',        'PDFTrailer')
+    _alias( Document, 'prev_location',      'previousLocation')
+    _alias( Document, 'resolve_link')
+    _alias( Document, 'search_page_for')
+    _alias( Document, 'set_language')
+    _alias( Document, 'set_metadata')
+    _alias( Document, 'set_toc', 'setToC')
+    _alias( Document, 'set_xml_metadata')
+    _alias( Document, 'update_object')
+    _alias( Document, 'update_stream')
+    _alias( Document, 'xref_is_stream',     'isStream')
+    _alias( Document, 'xref_length')
+    _alias( Document, 'xref_object')
+    _alias( Document, 'xref_stream')
+    _alias( Document, 'xref_stream_raw')
+    _alias( Document, 'xref_xml_metadata',  'metadataXML')
+    _alias( IRect, 'get_area')
+    _alias( IRect, 'get_area',              'getRectArea')
+    _alias( IRect, 'include_point')
+    _alias( IRect, 'include_rect')
+    _alias( IRect, 'is_empty')
+    _alias( IRect, 'is_infinite')
+    _alias( Link, 'is_external')
+    _alias( Link, 'set_border')
+    _alias( Link, 'set_colors')
+    _alias( Matrix, 'is_rectilinear')
+    _alias( Matrix, 'prerotate',            'preRotate')
+    _alias( Matrix, 'prescale',             'preScale')
+    _alias( Matrix, 'preshear',             'preShear')
+    _alias( Matrix, 'pretranslate',         'preTranslate')
+    _alias( None, 'get_pdf_now',            'getPDFnow')
+    _alias( None, 'get_pdf_str',            'getPDFstr')
+    _alias( None, 'get_text_length')
+    _alias( None, 'get_text_length',        'getTextlength')
+    _alias( None, 'image_profile',          'ImageProperties')
+    _alias( None, 'paper_rect',             'PaperRect')
+    _alias( None, 'paper_size',             'PaperSize')
+    _alias( None, 'paper_sizes')
+    _alias( None, 'planish_line')
+    _alias( Outline, 'is_external')
+    _alias( Outline, 'is_open')
+    _alias( Page, 'add_caret_annot')
+    _alias( Page, 'add_circle_annot')
+    _alias( Page, 'add_file_annot')
+    _alias( Page, 'add_freetext_annot')
+    _alias( Page, 'add_highlight_annot')
+    _alias( Page, 'add_ink_annot')
+    _alias( Page, 'add_line_annot')
+    _alias( Page, 'add_polygon_annot')
+    _alias( Page, 'add_polyline_annot')
+    _alias( Page, 'add_rect_annot')
+    _alias( Page, 'add_redact_annot')
+    _alias( Page, 'add_squiggly_annot')
+    _alias( Page, 'add_stamp_annot')
+    _alias( Page, 'add_strikeout_annot')
+    _alias( Page, 'add_text_annot')
+    _alias( Page, 'add_underline_annot')
+    _alias( Page, 'add_widget')
+    _alias( Page, 'clean_contents')
+    _alias( Page, 'cropbox',                'CropBox')
+    _alias( Page, 'cropbox_position',       'CropBoxPosition')
+    _alias( Page, 'delete_annot')
+    _alias( Page, 'delete_link')
+    _alias( Page, 'delete_widget')
+    _alias( Page, 'derotation_matrix')
+    _alias( Page, 'draw_bezier')
+    _alias( Page, 'draw_circle')
+    _alias( Page, 'draw_curve')
+    _alias( Page, 'draw_line')
+    _alias( Page, 'draw_oval')
+    _alias( Page, 'draw_polyline')
+    _alias( Page, 'draw_quad')
+    _alias( Page, 'draw_rect')
+    _alias( Page, 'draw_sector')
+    _alias( Page, 'draw_squiggle')
+    _alias( Page, 'draw_zigzag')
+    _alias( Page, 'first_annot')
+    _alias( Page, 'first_link')
+    _alias( Page, 'first_widget')
+    _alias( Page, 'get_contents')
+    _alias( Page, 'get_displaylist',        'getDisplayList')
+    _alias( Page, 'get_drawings')
+    _alias( Page, 'get_fonts',              'getFontList')
+    _alias( Page, 'get_image_bbox')
+    _alias( Page, 'get_images',             'getImageList')
+    _alias( Page, 'get_links')
+    _alias( Page, 'get_pixmap')
+    _alias( Page, 'get_svg_image',          'getSVGimage')
+    _alias( Page, 'get_text')
+    _alias( Page, 'get_text_blocks')
+    _alias( Page, 'get_text_words')
+    _alias( Page, 'get_textbox')
+    _alias( Page, 'get_textpage',           'getTextPage')
+    _alias( Page, 'insert_font')
+    _alias( Page, 'insert_image')
+    _alias( Page, 'insert_link')
+    _alias( Page, 'insert_text')
+    _alias( Page, 'insert_textbox')
+    _alias( Page, 'is_wrapped',             '_isWrapped')
+    _alias( Page, 'load_annot')
+    _alias( Page, 'load_links')
+    _alias( Page, 'mediabox',               'MediaBox')
+    _alias( Page, 'mediabox_size',          'MediaBoxSize')
+    _alias( Page, 'new_shape')
+    _alias( Page, 'read_contents')
+    _alias( Page, 'rotation_matrix')
+    _alias( Page, 'search_for')
+    _alias( Page, 'set_cropbox',            'setCropBox')
+    _alias( Page, 'set_mediabox',           'setMediaBox')
+    _alias( Page, 'set_rotation')
+    _alias( Page, 'show_pdf_page',          'showPDFpage')
+    _alias( Page, 'transformation_matrix')
+    _alias( Page, 'update_link')
+    _alias( Page, 'wrap_contents')
+    _alias( Page, 'write_text')
+    _alias( Pixmap, 'clear_with')
+    _alias( Pixmap, 'copy',                 'copyPixmap')
+    _alias( Pixmap, 'gamma_with')
+    _alias( Pixmap, 'invert_irect',         'invertIRect')
+    _alias( Pixmap, 'pil_save',             'pillowWrite')
+    _alias( Pixmap, 'pil_tobytes',          'pillowData')
+    _alias( Pixmap, 'save',                 'writeImage')
+    _alias( Pixmap, 'save',                 'writePNG')
+    _alias( Pixmap, 'set_alpha')
+    _alias( Pixmap, 'set_dpi',              'setResolution')
+    _alias( Pixmap, 'set_origin')
+    _alias( Pixmap, 'set_pixel')
+    _alias( Pixmap, 'set_rect')
+    _alias( Pixmap, 'tint_with')
+    _alias( Pixmap, 'tobytes',              'getImageData')
+    _alias( Pixmap, 'tobytes',              'getPNGData')
+    _alias( Pixmap, 'tobytes',              'getPNGdata')
+    _alias( Quad, 'is_convex')
+    _alias( Quad, 'is_empty')
+    _alias( Quad, 'is_rectangular')
+    _alias( Rect, 'get_area')
+    _alias( Rect, 'get_area',               'getRectArea')
+    _alias( Rect, 'include_point')
+    _alias( Rect, 'include_rect')
+    _alias( Rect, 'is_empty')
+    _alias( Rect, 'is_infinite')
+    _alias( TextWriter, 'fill_textbox')
+    _alias( TextWriter, 'write_text')
+    _alias( utils.Shape, 'draw_bezier')
+    _alias( utils.Shape, 'draw_circle')
+    _alias( utils.Shape, 'draw_curve')
+    _alias( utils.Shape, 'draw_line')
+    _alias( utils.Shape, 'draw_oval')
+    _alias( utils.Shape, 'draw_polyline')
+    _alias( utils.Shape, 'draw_quad')
+    _alias( utils.Shape, 'draw_rect')
+    _alias( utils.Shape, 'draw_sector')
+    _alias( utils.Shape, 'draw_squiggle')
+    _alias( utils.Shape, 'draw_zigzag')
+    _alias( utils.Shape, 'insert_text')
+    _alias( utils.Shape, 'insert_textbox')
+
+if 0:
+    restore_aliases()
+
+__version__ = VersionBind
+__doc__ = (
+        f'PyMuPDF {VersionBind}: Python bindings for the MuPDF {VersionFitz} library (rebased implementation).\n'
+        f'Python {sys.version_info[0]}.{sys.version_info[1]} running on {sys.platform} ({64 if sys.maxsize > 2**32 else 32}-bit).\n'
+        )
diff --git a/lib/python3.1/site-packages/pymupdf/__main__.py b/lib/python3.1/site-packages/pymupdf/__main__.py
new file mode 100644
index 0000000000000000000000000000000000000000..35914d6c724d4382b9ad65239a270d07ea33e3a9
--- /dev/null
+++ b/lib/python3.1/site-packages/pymupdf/__main__.py
@@ -0,0 +1,1140 @@
+# -----------------------------------------------------------------------------
+# Copyright 2020-2022, Harald Lieder, mailto:harald.lieder@outlook.com
+# License: GNU AFFERO GPL 3.0, https://www.gnu.org/licenses/agpl-3.0.html
+# Part of "PyMuPDF", Python bindings for "MuPDF" (http://mupdf.com), a
+# lightweight PDF, XPS, and E-book viewer, renderer and toolkit which is
+# maintained and developed by Artifex Software, Inc. https://artifex.com.
+# -----------------------------------------------------------------------------
+import argparse
+import bisect
+import os
+import sys
+import statistics
+from typing import Dict, List, Set
+
+from . import pymupdf
+
+def mycenter(x):
+    return (" %s " % x).center(75, "-")
+
+
+def recoverpix(doc, item):
+    """Return image for a given XREF."""
+    x = item[0]  # xref of PDF image
+    s = item[1]  # xref of its /SMask
+    if s == 0:  # no smask: use direct image output
+        return doc.extract_image(x)
+
+    def getimage(pix):
+        if pix.colorspace.n != 4:
+            return pix
+        tpix = pymupdf.Pixmap(pymupdf.csRGB, pix)
+        return tpix
+
+    # we need to reconstruct the alpha channel with the smask
+    pix1 = pymupdf.Pixmap(doc, x)
+    pix2 = pymupdf.Pixmap(doc, s)  # create pixmap of the /SMask entry
+
+    """Sanity check:
+    - both pixmaps must have the same rectangle
+    - both pixmaps must have alpha=0
+    - pix2 must consist of 1 byte per pixel
+    """
+    if not (pix1.irect == pix2.irect and pix1.alpha == pix2.alpha == 0 and pix2.n == 1):
+        pymupdf.message("Warning: unsupported /SMask %i for %i:" % (s, x))
+        pymupdf.message(pix2)
+        pix2 = None
+        return getimage(pix1)  # return the pixmap as is
+
+    pix = pymupdf.Pixmap(pix1)  # copy of pix1, with an alpha channel added
+    pix.set_alpha(pix2.samples)  # treat pix2.samples as the alpha values
+    pix1 = pix2 = None  # free temp pixmaps
+
+    # we may need to adjust something for CMYK pixmaps here:
+    return getimage(pix)
+
+
+def open_file(filename, password, show=False, pdf=True):
+    """Open and authenticate a document."""
+    doc = pymupdf.open(filename)
+    if not doc.is_pdf and pdf is True:
+        sys.exit("this command supports PDF files only")
+    rc = -1
+    if not doc.needs_pass:
+        return doc
+    if password:
+        rc = doc.authenticate(password)
+        if not rc:
+            sys.exit("authentication unsuccessful")
+        if show is True:
+            pymupdf.message("authenticated as %s" % "owner" if rc > 2 else "user")
+    else:
+        sys.exit("'%s' requires a password" % doc.name)
+    return doc
+
+
+def print_dict(item):
+    """Print a Python dictionary."""
+    l = max([len(k) for k in item.keys()]) + 1
+    for k, v in item.items():
+        msg = "%s: %s" % (k.rjust(l), v)
+        pymupdf.message(msg)
+
+
+def print_xref(doc, xref):
+    """Print an object given by XREF number.
+
+    Simulate the PDF source in "pretty" format.
+    For a stream also print its size.
+    """
+    pymupdf.message("%i 0 obj" % xref)
+    xref_str = doc.xref_object(xref)
+    pymupdf.message(xref_str)
+    if doc.xref_is_stream(xref):
+        temp = xref_str.split()
+        try:
+            idx = temp.index("/Length") + 1
+            size = temp[idx]
+            if size.endswith("0 R"):
+                size = "unknown"
+        except Exception:
+            size = "unknown"
+        pymupdf.message("stream\n...%s bytes" % size)
+        pymupdf.message("endstream")
+    pymupdf.message("endobj")
+
+
+def get_list(rlist, limit, what="page"):
+    """Transform a page / xref specification into a list of integers.
+
+    Args
+    ----
+        rlist: (str) the specification
+        limit: maximum number, i.e. number of pages, number of objects
+        what: a string to be used in error messages
+    Returns
+    -------
+        A list of integers representing the specification.
+    """
+    N = str(limit - 1)
+    rlist = rlist.replace("N", N).replace(" ", "")
+    rlist_arr = rlist.split(",")
+    out_list = []
+    for seq, item in enumerate(rlist_arr):
+        n = seq + 1
+        if item.isdecimal():  # a single integer
+            i = int(item)
+            if 1 <= i < limit:
+                out_list.append(int(item))
+            else:
+                sys.exit("bad %s specification at item %i" % (what, n))
+            continue
+        try:  # this must be a range now, and all of the following must work:
+            i1, i2 = item.split("-")  # will fail if not 2 items produced
+            i1 = int(i1)  # will fail on non-integers
+            i2 = int(i2)
+        except Exception:
+            sys.exit("bad %s range specification at item %i" % (what, n))
+
+        if not (1 <= i1 < limit and 1 <= i2 < limit):
+            sys.exit("bad %s range specification at item %i" % (what, n))
+
+        if i1 == i2:  # just in case: a range of equal numbers
+            out_list.append(i1)
+            continue
+
+        if i1 < i2:  # first less than second
+            out_list += list(range(i1, i2 + 1))
+        else:  # first larger than second
+            out_list += list(range(i1, i2 - 1, -1))
+
+    return out_list
+
+
+def show(args):
+    doc = open_file(args.input, args.password, True)
+    size = os.path.getsize(args.input) / 1024
+    flag = "KB"
+    if size > 1000:
+        size /= 1024
+        flag = "MB"
+    size = round(size, 1)
+    meta = doc.metadata # pylint: disable=no-member
+    pymupdf.message(
+        "'%s', pages: %i, objects: %i, %g %s, %s, encryption: %s"
+        % (
+            args.input,
+            doc.page_count,
+            doc.xref_length() - 1,
+            size,
+            flag,
+            meta["format"],
+            meta["encryption"],
+        )
+    )
+    n = doc.is_form_pdf
+    if n > 0:
+        s = doc.get_sigflags()
+        pymupdf.message(
+            "document contains %i root form fields and is %ssigned"
+            % (n, "not " if s != 3 else "")
+        )
+    n = doc.embfile_count()
+    if n > 0:
+        pymupdf.message("document contains %i embedded files" % n)
+    pymupdf.message()
+    if args.catalog:
+        pymupdf.message(mycenter("PDF catalog"))
+        xref = doc.pdf_catalog()
+        print_xref(doc, xref)
+        pymupdf.message()
+    if args.metadata:
+        pymupdf.message(mycenter("PDF metadata"))
+        print_dict(doc.metadata)    # pylint: disable=no-member
+        pymupdf.message()
+    if args.xrefs:
+        pymupdf.message(mycenter("object information"))
+        xrefl = get_list(args.xrefs, doc.xref_length(), what="xref")
+        for xref in xrefl:
+            print_xref(doc, xref)
+            pymupdf.message()
+    if args.pages:
+        pymupdf.message(mycenter("page information"))
+        pagel = get_list(args.pages, doc.page_count + 1)
+        for pno in pagel:
+            n = pno - 1
+            xref = doc.page_xref(n)
+            pymupdf.message("Page %i:" % pno)
+            print_xref(doc, xref)
+            pymupdf.message()
+    if args.trailer:
+        pymupdf.message(mycenter("PDF trailer"))
+        pymupdf.message(doc.pdf_trailer())
+        pymupdf.message()
+    doc.close()
+
+
+def clean(args):
+    doc = open_file(args.input, args.password, pdf=True)
+    encryption = args.encryption
+    encrypt = ("keep", "none", "rc4-40", "rc4-128", "aes-128", "aes-256").index(
+        encryption
+    )
+
+    if not args.pages:  # simple cleaning
+        doc.save(
+            args.output,
+            garbage=args.garbage,
+            deflate=args.compress,
+            pretty=args.pretty,
+            clean=args.sanitize,
+            ascii=args.ascii,
+            linear=args.linear,
+            encryption=encrypt,
+            owner_pw=args.owner,
+            user_pw=args.user,
+            permissions=args.permission,
+        )
+        return
+
+    # create sub document from page numbers
+    pages = get_list(args.pages, doc.page_count + 1)
+    outdoc = pymupdf.open()
+    for pno in pages:
+        n = pno - 1
+        outdoc.insert_pdf(doc, from_page=n, to_page=n)
+    outdoc.save(
+        args.output,
+        garbage=args.garbage,
+        deflate=args.compress,
+        pretty=args.pretty,
+        clean=args.sanitize,
+        ascii=args.ascii,
+        linear=args.linear,
+        encryption=encrypt,
+        owner_pw=args.owner,
+        user_pw=args.user,
+        permissions=args.permission,
+    )
+    doc.close()
+    outdoc.close()
+    return
+
+
+def doc_join(args):
+    """Join pages from several PDF documents."""
+    doc_list = args.input  # a list of input PDFs
+    doc = pymupdf.open()  # output PDF
+    for src_item in doc_list:  # process one input PDF
+        src_list = src_item.split(",")
+        password = src_list[1] if len(src_list) > 1 else None
+        src = open_file(src_list[0], password, pdf=True)
+        pages = ",".join(src_list[2:])  # get 'pages' specifications
+        if pages:  # if anything there, retrieve a list of desired pages
+            page_list = get_list(",".join(src_list[2:]), src.page_count + 1)
+        else:  # take all pages
+            page_list = range(1, src.page_count + 1)
+        for i in page_list:
+            doc.insert_pdf(src, from_page=i - 1, to_page=i - 1)  # copy each source page
+        src.close()
+
+    doc.save(args.output, garbage=4, deflate=True)
+    doc.close()
+
+
+def embedded_copy(args):
+    """Copy embedded files between PDFs."""
+    doc = open_file(args.input, args.password, pdf=True)
+    if not doc.can_save_incrementally() and (
+        not args.output or args.output == args.input
+    ):
+        sys.exit("cannot save PDF incrementally")
+    src = open_file(args.source, args.pwdsource)
+    names = set(args.name) if args.name else set()
+    src_names = set(src.embfile_names())
+    if names:
+        if not names <= src_names:
+            sys.exit("not all names are contained in source")
+    else:
+        names = src_names
+    if not names:
+        sys.exit("nothing to copy")
+    intersect = names & set(doc.embfile_names())  # any equal name already in target?
+    if intersect:
+        sys.exit("following names already exist in receiving PDF: %s" % str(intersect))
+
+    for item in names:
+        info = src.embfile_info(item)
+        buff = src.embfile_get(item)
+        doc.embfile_add(
+            item,
+            buff,
+            filename=info["filename"],
+            ufilename=info["ufilename"],
+            desc=info["desc"],
+        )
+        pymupdf.message("copied entry '%s' from '%s'" % (item, src.name))
+    src.close()
+    if args.output and args.output != args.input:
+        doc.save(args.output, garbage=3)
+    else:
+        doc.saveIncr()
+    doc.close()
+
+
+def embedded_del(args):
+    """Delete an embedded file entry."""
+    doc = open_file(args.input, args.password, pdf=True)
+    if not doc.can_save_incrementally() and (
+        not args.output or args.output == args.input
+    ):
+        sys.exit("cannot save PDF incrementally")
+
+    try:
+        doc.embfile_del(args.name)
+    except (ValueError, pymupdf.mupdf.FzErrorBase) as e:
+        sys.exit(f'no such embedded file {args.name!r}: {e}')
+    if not args.output or args.output == args.input:
+        doc.saveIncr()
+    else:
+        doc.save(args.output, garbage=1)
+    doc.close()
+
+
+def embedded_get(args):
+    """Retrieve contents of an embedded file."""
+    doc = open_file(args.input, args.password, pdf=True)
+    try:
+        stream = doc.embfile_get(args.name)
+        d = doc.embfile_info(args.name)
+    except (ValueError, pymupdf.mupdf.FzErrorBase) as e:
+        sys.exit(f'no such embedded file {args.name!r}: {e}')
+    filename = args.output if args.output else d["filename"]
+    with open(filename, "wb") as output:
+        output.write(stream)
+    pymupdf.message("saved entry '%s' as '%s'" % (args.name, filename))
+    doc.close()
+
+
+def embedded_add(args):
+    """Insert a new embedded file."""
+    doc = open_file(args.input, args.password, pdf=True)
+    if not doc.can_save_incrementally() and (
+        args.output is None or args.output == args.input
+    ):
+        sys.exit("cannot save PDF incrementally")
+
+    try:
+        doc.embfile_del(args.name)
+        sys.exit("entry '%s' already exists" % args.name)
+    except Exception:
+        pass
+
+    if not os.path.exists(args.path) or not os.path.isfile(args.path):
+        sys.exit("no such file '%s'" % args.path)
+    with open(args.path, "rb") as f:
+        stream = f.read()
+    filename = args.path
+    ufilename = filename
+    if not args.desc:
+        desc = filename
+    else:
+        desc = args.desc
+    doc.embfile_add(
+        args.name, stream, filename=filename, ufilename=ufilename, desc=desc
+    )
+    if not args.output or args.output == args.input:
+        doc.saveIncr()
+    else:
+        doc.save(args.output, garbage=3)
+    doc.close()
+
+
+def embedded_upd(args):
+    """Update contents or metadata of an embedded file."""
+    doc = open_file(args.input, args.password, pdf=True)
+    if not doc.can_save_incrementally() and (
+        args.output is None or args.output == args.input
+    ):
+        sys.exit("cannot save PDF incrementally")
+
+    try:
+        doc.embfile_info(args.name)
+    except Exception:
+        sys.exit("no such embedded file '%s'" % args.name)
+
+    if (
+        args.path is not None
+        and os.path.exists(args.path)
+        and os.path.isfile(args.path)
+    ):
+        with open(args.path, "rb") as f:
+            stream = f.read()
+    else:
+        stream = None
+
+    if args.filename:
+        filename = args.filename
+    else:
+        filename = None
+
+    if args.ufilename:
+        ufilename = args.ufilename
+    elif args.filename:
+        ufilename = args.filename
+    else:
+        ufilename = None
+
+    if args.desc:
+        desc = args.desc
+    else:
+        desc = None
+
+    doc.embfile_upd(
+        args.name, stream, filename=filename, ufilename=ufilename, desc=desc
+    )
+    if args.output is None or args.output == args.input:
+        doc.saveIncr()
+    else:
+        doc.save(args.output, garbage=3)
+    doc.close()
+
+
+def embedded_list(args):
+    """List embedded files."""
+    doc = open_file(args.input, args.password, pdf=True)
+    names = doc.embfile_names()
+    if args.name is not None:
+        if args.name not in names:
+            sys.exit("no such embedded file '%s'" % args.name)
+        else:
+            pymupdf.message()
+            pymupdf.message(
+                "printing 1 of %i embedded file%s:"
+                % (len(names), "s" if len(names) > 1 else "")
+            )
+            pymupdf.message()
+            print_dict(doc.embfile_info(args.name))
+            pymupdf.message()
+            return
+    if not names:
+        pymupdf.message("'%s' contains no embedded files" % doc.name)
+        return
+    if len(names) > 1:
+        msg = "'%s' contains the following %i embedded files" % (doc.name, len(names))
+    else:
+        msg = "'%s' contains the following embedded file" % doc.name
+    pymupdf.message(msg)
+    pymupdf.message()
+    for name in names:
+        if not args.detail:
+            pymupdf.message(name)
+            continue
+        _ = doc.embfile_info(name)
+        print_dict(doc.embfile_info(name))
+        pymupdf.message()
+    doc.close()
+
+
+def extract_objects(args):
+    """Extract images and / or fonts from a PDF."""
+    if not args.fonts and not args.images:
+        sys.exit("neither fonts nor images requested")
+    doc = open_file(args.input, args.password, pdf=True)
+
+    if args.pages:
+        pages = get_list(args.pages, doc.page_count + 1)
+    else:
+        pages = range(1, doc.page_count + 1)
+
+    if not args.output:
+        out_dir = os.path.abspath(os.curdir)
+    else:
+        out_dir = args.output
+        if not (os.path.exists(out_dir) and os.path.isdir(out_dir)):
+            sys.exit("output directory %s does not exist" % out_dir)
+
+    font_xrefs = set()  # already saved fonts
+    image_xrefs = set()  # already saved images
+
+    for pno in pages:
+        if args.fonts:
+            itemlist = doc.get_page_fonts(pno - 1)
+            for item in itemlist:
+                xref = item[0]
+                if xref not in font_xrefs:
+                    font_xrefs.add(xref)
+                    fontname, ext, _, buffer = doc.extract_font(xref)
+                    if ext == "n/a" or not buffer:
+                        continue
+                    outname = os.path.join(
+                        out_dir, f"{fontname.replace(' ', '-')}-{xref}.{ext}"
+                    )
+                    with open(outname, "wb") as outfile:
+                        outfile.write(buffer)
+                    buffer = None
+        if args.images:
+            itemlist = doc.get_page_images(pno - 1)
+            for item in itemlist:
+                xref = item[0]
+                if xref not in image_xrefs:
+                    image_xrefs.add(xref)
+                    pix = recoverpix(doc, item)
+                    if type(pix) is dict:
+                        ext = pix["ext"]
+                        imgdata = pix["image"]
+                        outname = os.path.join(out_dir, "img-%i.%s" % (xref, ext))
+                        with open(outname, "wb") as outfile:
+                            outfile.write(imgdata)
+                    else:
+                        outname = os.path.join(out_dir, "img-%i.png" % xref)
+                        pix2 = (
+                            pix
+                            if pix.colorspace.n < 4
+                            else pymupdf.Pixmap(pymupdf.csRGB, pix)
+                        )
+                        pix2.save(outname)
+
+    if args.fonts:
+        pymupdf.message("saved %i fonts to '%s'" % (len(font_xrefs), out_dir))
+    if args.images:
+        pymupdf.message("saved %i images to '%s'" % (len(image_xrefs), out_dir))
+    doc.close()
+
+
+def page_simple(page, textout, GRID, fontsize, noformfeed, skip_empty, flags):
+    eop = b"\n" if noformfeed else bytes([12])
+    text = page.get_text("text", flags=flags)
+    if not text:
+        if not skip_empty:
+            textout.write(eop)  # write formfeed
+        return
+    textout.write(text.encode("utf8", errors="surrogatepass"))
+    textout.write(eop)
+    return
+
+
+def page_blocksort(page, textout, GRID, fontsize, noformfeed, skip_empty, flags):
+    eop = b"\n" if noformfeed else bytes([12])
+    blocks = page.get_text("blocks", flags=flags)
+    if blocks == []:
+        if not skip_empty:
+            textout.write(eop)  # write formfeed
+        return
+    blocks.sort(key=lambda b: (b[3], b[0]))
+    for b in blocks:
+        textout.write(b[4].encode("utf8", errors="surrogatepass"))
+    textout.write(eop)
+    return
+
+
+def page_layout(page, textout, GRID, fontsize, noformfeed, skip_empty, flags):
+    eop = b"\n" if noformfeed else bytes([12])
+
+    # --------------------------------------------------------------------
+    def find_line_index(values: List[int], value: int) -> int:
+        """Find the right row coordinate.
+
+        Args:
+            values: (list) y-coordinates of rows.
+            value: (int) lookup for this value (y-origin of char).
+        Returns:
+            y-ccordinate of appropriate line for value.
+        """
+        i = bisect.bisect_right(values, value)
+        if i:
+            return values[i - 1]
+        raise RuntimeError("Line for %g not found in %s" % (value, values))
+
+    # --------------------------------------------------------------------
+    def curate_rows(rows: Set[int], GRID) -> List:
+        rows = list(rows)
+        rows.sort()  # sort ascending
+        nrows = [rows[0]]
+        for h in rows[1:]:
+            if h >= nrows[-1] + GRID:  # only keep significant differences
+                nrows.append(h)
+        return nrows  # curated list of line bottom coordinates
+
+    def process_blocks(blocks: List[Dict], page: pymupdf.Page):
+        rows = set()
+        page_width = page.rect.width
+        page_height = page.rect.height
+        rowheight = page_height
+        left = page_width
+        right = 0
+        chars = []
+        for block in blocks:
+            for line in block["lines"]:
+                if line["dir"] != (1, 0):  # ignore non-horizontal text
+                    continue
+                x0, y0, x1, y1 = line["bbox"]
+                if y1 < 0 or y0 > page.rect.height:  # ignore if outside CropBox
+                    continue
+                # upd row height
+                height = y1 - y0
+
+                if rowheight > height:
+                    rowheight = height
+                for span in line["spans"]:
+                    if span["size"] <= fontsize:
+                        continue
+                    for c in span["chars"]:
+                        x0, _, x1, _ = c["bbox"]
+                        cwidth = x1 - x0
+                        ox, oy = c["origin"]
+                        oy = int(round(oy))
+                        rows.add(oy)
+                        ch = c["c"]
+                        if left > ox and ch != " ":
+                            left = ox  # update left coordinate
+                        if right < x1:
+                            right = x1  # update right coordinate
+                        # handle ligatures:
+                        if cwidth == 0 and chars != []:  # potential ligature
+                            old_ch, old_ox, old_oy, old_cwidth = chars[-1]
+                            if old_oy == oy:  # ligature
+                                if old_ch != chr(0xFB00):  # previous "ff" char lig?
+                                    lig = joinligature(old_ch + ch)  # no
+                                # convert to one of the 3-char ligatures:
+                                elif ch == "i":
+                                    lig = chr(0xFB03)  # "ffi"
+                                elif ch == "l":
+                                    lig = chr(0xFB04)  # "ffl"
+                                else:  # something wrong, leave old char in place
+                                    lig = old_ch
+                                chars[-1] = (lig, old_ox, old_oy, old_cwidth)
+                                continue
+                        chars.append((ch, ox, oy, cwidth))  # all chars on page
+        return chars, rows, left, right, rowheight
+
+    def joinligature(lig: str) -> str:
+        """Return ligature character for a given pair / triple of characters.
+
+        Args:
+            lig: (str) 2/3 characters, e.g. "ff"
+        Returns:
+            Ligature, e.g. "ff" -> chr(0xFB00)
+        """
+
+        if lig == "ff":
+            return chr(0xFB00)
+        elif lig == "fi":
+            return chr(0xFB01)
+        elif lig == "fl":
+            return chr(0xFB02)
+        elif lig == "ffi":
+            return chr(0xFB03)
+        elif lig == "ffl":
+            return chr(0xFB04)
+        elif lig == "ft":
+            return chr(0xFB05)
+        elif lig == "st":
+            return chr(0xFB06)
+        return lig
+
+    # --------------------------------------------------------------------
+    def make_textline(left, slot, minslot, lchars):
+        """Produce the text of one output line.
+
+        Args:
+            left: (float) left most coordinate used on page
+            slot: (float) avg width of one character in any font in use.
+            minslot: (float) min width for the characters in this line.
+            chars: (list[tuple]) characters of this line.
+        Returns:
+            text: (str) text string for this line
+        """
+        text = ""  # we output this
+        old_char = ""
+        old_x1 = 0  # end coordinate of last char
+        old_ox = 0  # x-origin of last char
+        if minslot <= pymupdf.EPSILON:
+            raise RuntimeError("program error: minslot too small = %g" % minslot)
+
+        for c in lchars:  # loop over characters
+            char, ox, _, cwidth = c
+            ox = ox - left  # its (relative) start coordinate
+            x1 = ox + cwidth  # ending coordinate
+
+            # eliminate overprint effect
+            if old_char == char and ox - old_ox <= cwidth * 0.2:
+                continue
+
+            # omit spaces overlapping previous char
+            if char == " " and (old_x1 - ox) / cwidth > 0.8:
+                continue
+
+            old_char = char
+            # close enough to previous?
+            if ox < old_x1 + minslot:  # assume char adjacent to previous
+                text += char  # append to output
+                old_x1 = x1  # new end coord
+                old_ox = ox  # new origin.x
+                continue
+
+            # else next char starts after some gap:
+            # fill in right number of spaces, so char is positioned
+            # in the right slot of the line
+            if char == " ":  # rest relevant for non-space only
+                continue
+            delta = int(ox / slot) - len(text)
+            if ox > old_x1 and delta > 1:
+                text += " " * delta
+            # now append char
+            text += char
+            old_x1 = x1  # new end coordinate
+            old_ox = ox  # new origin
+        return text.rstrip()
+
+    # extract page text by single characters ("rawdict")
+    blocks = page.get_text("rawdict", flags=flags)["blocks"]
+    chars, rows, left, right, rowheight = process_blocks(blocks, page)
+
+    if chars == []:
+        if not skip_empty:
+            textout.write(eop)  # write formfeed
+        return
+    # compute list of line coordinates - ignoring small (GRID) differences
+    rows = curate_rows(rows, GRID)
+
+    # sort all chars by x-coordinates, so every line will receive char info,
+    # sorted from left to right.
+    chars.sort(key=lambda c: c[1])
+
+    # populate the lines with their char info
+    lines = {}  # key: y1-ccordinate, value: char list
+    for c in chars:
+        _, _, oy, _ = c
+        y = find_line_index(rows, oy)  # y-coord of the right line
+        lchars = lines.get(y, [])  # read line chars so far
+        lchars.append(c)  # append this char
+        lines[y] = lchars  # write back to line
+
+    # ensure line coordinates are ascending
+    keys = list(lines.keys())
+    keys.sort()
+
+    # -------------------------------------------------------------------------
+    # Compute "char resolution" for the page: the char width corresponding to
+    # 1 text char position on output - call it 'slot'.
+    # For each line, compute median of its char widths. The minimum across all
+    # lines is 'slot'.
+    # The minimum char width of each line is used to determine if spaces must
+    # be inserted in between two characters.
+    # -------------------------------------------------------------------------
+    slot = right - left
+    minslots = {}
+    for k in keys:
+        lchars = lines[k]
+        ccount = len(lchars)
+        if ccount < 2:
+            minslots[k] = 1
+            continue
+        widths = [c[3] for c in lchars]
+        widths.sort()
+        this_slot = statistics.median(widths)  # take median value
+        if this_slot < slot:
+            slot = this_slot
+        minslots[k] = widths[0]
+
+    # compute line advance in text output
+    rowheight = rowheight * (rows[-1] - rows[0]) / (rowheight * len(rows)) * 1.2
+    rowpos = rows[0]  # first line positioned here
+    textout.write(b"\n")
+    for k in keys:  # walk through the lines
+        while rowpos < k:  # honor distance between lines
+            textout.write(b"\n")
+            rowpos += rowheight
+        text = make_textline(left, slot, minslots[k], lines[k])
+        textout.write((text + "\n").encode("utf8", errors="surrogatepass"))
+        rowpos = k + rowheight
+
+    textout.write(eop)  # write formfeed
+
+
+def gettext(args):
+    doc = open_file(args.input, args.password, pdf=False)
+    pagel = get_list(args.pages, doc.page_count + 1)
+    output = args.output
+    if output is None:
+        filename, _ = os.path.splitext(doc.name)
+        output = filename + ".txt"
+    with open(output, "wb") as textout:
+        flags = pymupdf.TEXT_PRESERVE_LIGATURES | pymupdf.TEXT_PRESERVE_WHITESPACE
+        if args.convert_white:
+            flags ^= pymupdf.TEXT_PRESERVE_WHITESPACE
+        if args.noligatures:
+            flags ^= pymupdf.TEXT_PRESERVE_LIGATURES
+        if args.extra_spaces:
+            flags ^= pymupdf.TEXT_INHIBIT_SPACES
+        func = {
+            "simple": page_simple,
+            "blocks": page_blocksort,
+            "layout": page_layout,
+        }
+        for pno in pagel:
+            page = doc[pno - 1]
+            func[args.mode](
+                page,
+                textout,
+                args.grid,
+                args.fontsize,
+                args.noformfeed,
+                args.skip_empty,
+                flags=flags,
+            )
+
+
+def _internal(args):
+    pymupdf.message('This is from PyMuPDF message().')
+    pymupdf.log('This is from PyMuPDF log().')
+
+def main():
+    """Define command configurations."""
+    parser = argparse.ArgumentParser(
+        prog="pymupdf",
+        description=mycenter("Basic PyMuPDF Functions"),
+    )
+    subps = parser.add_subparsers(
+        title="Subcommands", help="Enter 'command -h' for subcommand specific help"
+    )
+
+    # -------------------------------------------------------------------------
+    # 'show' command
+    # -------------------------------------------------------------------------
+    ps_show = subps.add_parser("show", description=mycenter("display PDF information"))
+    ps_show.add_argument("input", type=str, help="PDF filename")
+    ps_show.add_argument("-password", help="password")
+    ps_show.add_argument("-catalog", action="store_true", help="show PDF catalog")
+    ps_show.add_argument("-trailer", action="store_true", help="show PDF trailer")
+    ps_show.add_argument("-metadata", action="store_true", help="show PDF metadata")
+    ps_show.add_argument(
+        "-xrefs", type=str, help="show selected objects, format: 1,5-7,N"
+    )
+    ps_show.add_argument(
+        "-pages", type=str, help="show selected pages, format: 1,5-7,50-N"
+    )
+    ps_show.set_defaults(func=show)
+
+    # -------------------------------------------------------------------------
+    # 'clean' command
+    # -------------------------------------------------------------------------
+    ps_clean = subps.add_parser(
+        "clean", description=mycenter("optimize PDF, or create sub-PDF if pages given")
+    )
+    ps_clean.add_argument("input", type=str, help="PDF filename")
+    ps_clean.add_argument("output", type=str, help="output PDF filename")
+    ps_clean.add_argument("-password", help="password")
+
+    ps_clean.add_argument(
+        "-encryption",
+        help="encryption method",
+        choices=("keep", "none", "rc4-40", "rc4-128", "aes-128", "aes-256"),
+        default="none",
+    )
+
+    ps_clean.add_argument("-owner", type=str, help="owner password")
+    ps_clean.add_argument("-user", type=str, help="user password")
+
+    ps_clean.add_argument(
+        "-garbage",
+        type=int,
+        help="garbage collection level",
+        choices=range(5),
+        default=0,
+    )
+
+    ps_clean.add_argument(
+        "-compress",
+        action="store_true",
+        default=False,
+        help="compress (deflate) output",
+    )
+
+    ps_clean.add_argument(
+        "-ascii", action="store_true", default=False, help="ASCII encode binary data"
+    )
+
+    ps_clean.add_argument(
+        "-linear",
+        action="store_true",
+        default=False,
+        help="format for fast web display",
+    )
+
+    ps_clean.add_argument(
+        "-permission", type=int, default=-1, help="integer with permission levels"
+    )
+
+    ps_clean.add_argument(
+        "-sanitize",
+        action="store_true",
+        default=False,
+        help="sanitize / clean contents",
+    )
+    ps_clean.add_argument(
+        "-pretty", action="store_true", default=False, help="prettify PDF structure"
+    )
+    ps_clean.add_argument(
+        "-pages", help="output selected pages pages, format: 1,5-7,50-N"
+    )
+    ps_clean.set_defaults(func=clean)
+
+    # -------------------------------------------------------------------------
+    # 'join' command
+    # -------------------------------------------------------------------------
+    ps_join = subps.add_parser(
+        "join",
+        description=mycenter("join PDF documents"),
+        epilog="specify each input as 'filename[,password[,pages]]'",
+    )
+    ps_join.add_argument("input", nargs="*", help="input filenames")
+    ps_join.add_argument("-output", required=True, help="output filename")
+    ps_join.set_defaults(func=doc_join)
+
+    # -------------------------------------------------------------------------
+    # 'extract' command
+    # -------------------------------------------------------------------------
+    ps_extract = subps.add_parser(
+        "extract", description=mycenter("extract images and fonts to disk")
+    )
+    ps_extract.add_argument("input", type=str, help="PDF filename")
+    ps_extract.add_argument("-images", action="store_true", help="extract images")
+    ps_extract.add_argument("-fonts", action="store_true", help="extract fonts")
+    ps_extract.add_argument(
+        "-output", help="folder to receive output, defaults to current"
+    )
+    ps_extract.add_argument("-password", help="password")
+    ps_extract.add_argument(
+        "-pages", type=str, help="consider these pages only, format: 1,5-7,50-N"
+    )
+    ps_extract.set_defaults(func=extract_objects)
+
+    # -------------------------------------------------------------------------
+    # 'embed-info'
+    # -------------------------------------------------------------------------
+    ps_show = subps.add_parser(
+        "embed-info", description=mycenter("list embedded files")
+    )
+    ps_show.add_argument("input", help="PDF filename")
+    ps_show.add_argument("-name", help="if given, report only this one")
+    ps_show.add_argument("-detail", action="store_true", help="detail information")
+    ps_show.add_argument("-password", help="password")
+    ps_show.set_defaults(func=embedded_list)
+
+    # -------------------------------------------------------------------------
+    # 'embed-add' command
+    # -------------------------------------------------------------------------
+    ps_embed_add = subps.add_parser(
+        "embed-add", description=mycenter("add embedded file")
+    )
+    ps_embed_add.add_argument("input", help="PDF filename")
+    ps_embed_add.add_argument("-password", help="password")
+    ps_embed_add.add_argument(
+        "-output", help="output PDF filename, incremental save if none"
+    )
+    ps_embed_add.add_argument("-name", required=True, help="name of new entry")
+    ps_embed_add.add_argument("-path", required=True, help="path to data for new entry")
+    ps_embed_add.add_argument("-desc", help="description of new entry")
+    ps_embed_add.set_defaults(func=embedded_add)
+
+    # -------------------------------------------------------------------------
+    # 'embed-del' command
+    # -------------------------------------------------------------------------
+    ps_embed_del = subps.add_parser(
+        "embed-del", description=mycenter("delete embedded file")
+    )
+    ps_embed_del.add_argument("input", help="PDF filename")
+    ps_embed_del.add_argument("-password", help="password")
+    ps_embed_del.add_argument(
+        "-output", help="output PDF filename, incremental save if none"
+    )
+    ps_embed_del.add_argument("-name", required=True, help="name of entry to delete")
+    ps_embed_del.set_defaults(func=embedded_del)
+
+    # -------------------------------------------------------------------------
+    # 'embed-upd' command
+    # -------------------------------------------------------------------------
+    ps_embed_upd = subps.add_parser(
+        "embed-upd",
+        description=mycenter("update embedded file"),
+        epilog="except '-name' all parameters are optional",
+    )
+    ps_embed_upd.add_argument("input", help="PDF filename")
+    ps_embed_upd.add_argument("-name", required=True, help="name of entry")
+    ps_embed_upd.add_argument("-password", help="password")
+    ps_embed_upd.add_argument(
+        "-output", help="Output PDF filename, incremental save if none"
+    )
+    ps_embed_upd.add_argument("-path", help="path to new data for entry")
+    ps_embed_upd.add_argument("-filename", help="new filename to store in entry")
+    ps_embed_upd.add_argument(
+        "-ufilename", help="new unicode filename to store in entry"
+    )
+    ps_embed_upd.add_argument("-desc", help="new description to store in entry")
+    ps_embed_upd.set_defaults(func=embedded_upd)
+
+    # -------------------------------------------------------------------------
+    # 'embed-extract' command
+    # -------------------------------------------------------------------------
+    ps_embed_extract = subps.add_parser(
+        "embed-extract", description=mycenter("extract embedded file to disk")
+    )
+    ps_embed_extract.add_argument("input", type=str, help="PDF filename")
+    ps_embed_extract.add_argument("-name", required=True, help="name of entry")
+    ps_embed_extract.add_argument("-password", help="password")
+    ps_embed_extract.add_argument(
+        "-output", help="output filename, default is stored name"
+    )
+    ps_embed_extract.set_defaults(func=embedded_get)
+
+    # -------------------------------------------------------------------------
+    # 'embed-copy' command
+    # -------------------------------------------------------------------------
+    ps_embed_copy = subps.add_parser(
+        "embed-copy", description=mycenter("copy embedded files between PDFs")
+    )
+    ps_embed_copy.add_argument("input", type=str, help="PDF to receive embedded files")
+    ps_embed_copy.add_argument("-password", help="password of input")
+    ps_embed_copy.add_argument(
+        "-output", help="output PDF, incremental save to 'input' if omitted"
+    )
+    ps_embed_copy.add_argument(
+        "-source", required=True, help="copy embedded files from here"
+    )
+    ps_embed_copy.add_argument("-pwdsource", help="password of 'source' PDF")
+    ps_embed_copy.add_argument(
+        "-name", nargs="*", help="restrict copy to these entries"
+    )
+    ps_embed_copy.set_defaults(func=embedded_copy)
+
+    # -------------------------------------------------------------------------
+    # 'textlayout' command
+    # -------------------------------------------------------------------------
+    ps_gettext = subps.add_parser(
+        "gettext", description=mycenter("extract text in various formatting modes")
+    )
+    ps_gettext.add_argument("input", type=str, help="input document filename")
+    ps_gettext.add_argument("-password", help="password for input document")
+    ps_gettext.add_argument(
+        "-mode",
+        type=str,
+        help="mode: simple, block sort, or layout (default)",
+        choices=("simple", "blocks", "layout"),
+        default="layout",
+    )
+    ps_gettext.add_argument(
+        "-pages",
+        type=str,
+        help="select pages, format: 1,5-7,50-N",
+        default="1-N",
+    )
+    ps_gettext.add_argument(
+        "-noligatures",
+        action="store_true",
+        help="expand ligature characters (default False)",
+        default=False,
+    )
+    ps_gettext.add_argument(
+        "-convert-white",
+        action="store_true",
+        help="convert whitespace characters to white (default False)",
+        default=False,
+    )
+    ps_gettext.add_argument(
+        "-extra-spaces",
+        action="store_true",
+        help="fill gaps with spaces (default False)",
+        default=False,
+    )
+    ps_gettext.add_argument(
+        "-noformfeed",
+        action="store_true",
+        help="write linefeeds, no formfeeds (default False)",
+        default=False,
+    )
+    ps_gettext.add_argument(
+        "-skip-empty",
+        action="store_true",
+        help="suppress pages with no text (default False)",
+        default=False,
+    )
+    ps_gettext.add_argument(
+        "-output",
+        help="store text in this file (default inputfilename.txt)",
+    )
+    ps_gettext.add_argument(
+        "-grid",
+        type=float,
+        help="merge lines if closer than this (default 2)",
+        default=2,
+    )
+    ps_gettext.add_argument(
+        "-fontsize",
+        type=float,
+        help="only include text with a larger fontsize (default 3)",
+        default=3,
+    )
+    ps_gettext.set_defaults(func=gettext)
+
+    # -------------------------------------------------------------------------
+    # '_internal' command
+    # -------------------------------------------------------------------------
+    ps_internal = subps.add_parser(
+        "internal", description=mycenter("internal testing")
+    )
+    ps_internal.set_defaults(func=_internal)
+
+    # -------------------------------------------------------------------------
+    # start program
+    # -------------------------------------------------------------------------
+    args = parser.parse_args()  # create parameter arguments class
+    if not hasattr(args, "func"):  # no function selected
+        parser.print_help()  # so print top level help
+    else:
+        args.func(args)  # execute requested command
+
+
+if __name__ == "__main__":
+    main()
diff --git a/lib/python3.1/site-packages/pymupdf/__pycache__/__main__.cpython-310.pyc b/lib/python3.1/site-packages/pymupdf/__pycache__/__main__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1b18fdb5c6dd52ecdcae212d0c07240c1c0c5222
Binary files /dev/null and b/lib/python3.1/site-packages/pymupdf/__pycache__/__main__.cpython-310.pyc differ
diff --git a/lib/python3.1/site-packages/pymupdf/__pycache__/_apply_pages.cpython-310.pyc b/lib/python3.1/site-packages/pymupdf/__pycache__/_apply_pages.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3b30a33f74372b140843ba1381f00173d0192f6a
Binary files /dev/null and b/lib/python3.1/site-packages/pymupdf/__pycache__/_apply_pages.cpython-310.pyc differ
diff --git a/lib/python3.1/site-packages/pymupdf/__pycache__/_build.cpython-310.pyc b/lib/python3.1/site-packages/pymupdf/__pycache__/_build.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e2e1db09fcf1a6db696efe66d229f9a425290a7d
Binary files /dev/null and b/lib/python3.1/site-packages/pymupdf/__pycache__/_build.cpython-310.pyc differ
diff --git a/lib/python3.1/site-packages/pymupdf/__pycache__/_wxcolors.cpython-310.pyc b/lib/python3.1/site-packages/pymupdf/__pycache__/_wxcolors.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..24aa7866d7a023cf24c613360cced601390eba42
Binary files /dev/null and b/lib/python3.1/site-packages/pymupdf/__pycache__/_wxcolors.cpython-310.pyc differ
diff --git a/lib/python3.1/site-packages/pymupdf/__pycache__/extra.cpython-310.pyc b/lib/python3.1/site-packages/pymupdf/__pycache__/extra.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f82a632aabec9538d4d708a984ce80468254e3d2
Binary files /dev/null and b/lib/python3.1/site-packages/pymupdf/__pycache__/extra.cpython-310.pyc differ
diff --git a/lib/python3.1/site-packages/pymupdf/__pycache__/pymupdf.cpython-310.pyc b/lib/python3.1/site-packages/pymupdf/__pycache__/pymupdf.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1356de59c74efbd7d845bb531da4f4e672054c79
Binary files /dev/null and b/lib/python3.1/site-packages/pymupdf/__pycache__/pymupdf.cpython-310.pyc differ
diff --git a/lib/python3.1/site-packages/pymupdf/__pycache__/table.cpython-310.pyc b/lib/python3.1/site-packages/pymupdf/__pycache__/table.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6e12a8340e06804ec9cd57d8beed41d38ab86d91
Binary files /dev/null and b/lib/python3.1/site-packages/pymupdf/__pycache__/table.cpython-310.pyc differ
diff --git a/lib/python3.1/site-packages/pymupdf/_apply_pages.py b/lib/python3.1/site-packages/pymupdf/_apply_pages.py
new file mode 100644
index 0000000000000000000000000000000000000000..0aae54c788ef7f79b49c19ea1765ee62fcc9976c
--- /dev/null
+++ b/lib/python3.1/site-packages/pymupdf/_apply_pages.py
@@ -0,0 +1,253 @@
+import multiprocessing
+import os
+import time
+
+import pymupdf
+
+
+# Support for concurrent processing of document pages.
+#
+
+class _worker_State:
+    pass
+_worker_state = _worker_State()
+
+
+def _worker_init(
+        path,
+        initfn,
+        initfn_args,
+        initfn_kwargs,
+        pagefn,
+        pagefn_args,
+        pagefn_kwargs,
+        stats,
+        ):
+    # pylint: disable=attribute-defined-outside-init
+    _worker_state.path = path
+    _worker_state.pagefn = pagefn
+    _worker_state.pagefn_args = pagefn_args
+    _worker_state.pagefn_kwargs = pagefn_kwargs
+    _worker_state.stats = stats
+    _worker_state.document = None
+    if initfn:
+        initfn(*initfn_args, **initfn_kwargs)
+
+
+def _stats_write(t, label):
+    t = time.time() - t
+    if t >= 10:
+        pymupdf.log(f'{os.getpid()=}: {t:2f}s: {label}.')
+
+
+def _worker_fn(page_number):
+    # Create Document from filename if we haven't already done so.
+    if not _worker_state.document:
+        if _worker_state.stats:
+            t = time.time()
+        _worker_state.document = pymupdf.Document(_worker_state.path)   # pylint: disable=attribute-defined-outside-init
+        if _worker_state.stats:
+            _stats_write(t, 'pymupdf.Document()')
+    
+    if _worker_state.stats:
+        t = time.time()
+    page = _worker_state.document[page_number]
+    if _worker_state.stats:
+        _stats_write(t, '_worker_state.document[page_number]')
+    
+    if _worker_state.stats:
+        t = time.time()
+    ret = _worker_state.pagefn(
+            page,
+            *_worker_state.pagefn_args,
+            **_worker_state.pagefn_kwargs,
+            )
+    if _worker_state.stats:
+        _stats_write(t, '_worker_state.pagefn()')
+    
+    return ret
+    
+
+def _multiprocessing(
+        path,
+        pages,
+        pagefn,
+        pagefn_args,
+        pagefn_kwargs,
+        initfn,
+        initfn_args,
+        initfn_kwargs,
+        concurrency,
+        stats,
+        ):
+    #print(f'_worker_mp(): {concurrency=}', flush=1)
+    with multiprocessing.Pool(
+            concurrency,
+            _worker_init,
+            (
+                path,
+                initfn, initfn_args, initfn_kwargs,
+                pagefn, pagefn_args, pagefn_kwargs,
+                stats,
+            ),
+            ) as pool:
+        result = pool.map_async(_worker_fn, pages)
+        return result.get()
+    
+
+def _fork(
+        path,
+        pages,
+        pagefn,
+        pagefn_args,
+        pagefn_kwargs,
+        initfn,
+        initfn_args,
+        initfn_kwargs,
+        concurrency,
+        stats,
+        ):
+    verbose = 0
+    if concurrency is None:
+        concurrency = multiprocessing.cpu_count()
+    # We write page numbers to `queue_down` and read `(page_num, text)` from
+    # `queue_up`. Workers each repeatedly read the next available page number
+    # from `queue_down`, extract the text and write it onto `queue_up`.
+    #
+    # This is better than pre-allocating a subset of pages to each worker
+    # because it ensures there will never be idle workers until we are near the
+    # end with fewer pages left than workers.
+    #
+    queue_down = multiprocessing.Queue()
+    queue_up = multiprocessing.Queue()
+    def childfn():
+        document = None
+        if verbose:
+            pymupdf.log(f'{os.getpid()=}: {initfn=} {initfn_args=}')
+        _worker_init(
+                path,
+                initfn,
+                initfn_args,
+                initfn_kwargs,
+                pagefn,
+                pagefn_args,
+                pagefn_kwargs,
+                stats,
+                )
+        while 1:
+            if verbose:
+                pymupdf.log(f'{os.getpid()=}: calling get().')
+            page_num = queue_down.get()
+            if verbose:
+                pymupdf.log(f'{os.getpid()=}: {page_num=}.')
+            if page_num is None:
+                break
+            try:
+                if not document:
+                    if stats:
+                        t = time.time()
+                    document = pymupdf.Document(path)
+                    if stats:
+                        _stats_write(t, 'pymupdf.Document(path)')
+                
+                if stats:
+                    t = time.time()
+                page = document[page_num]
+                if stats:
+                    _stats_write(t, 'document[page_num]')
+                
+                if verbose:
+                    pymupdf.log(f'{os.getpid()=}: {_worker_state=}')
+                
+                if stats:
+                    t = time.time()
+                ret = pagefn(
+                        page,
+                        *_worker_state.pagefn_args,
+                        **_worker_state.pagefn_kwargs,
+                        )
+                if stats:
+                    _stats_write(t, f'{page_num=} pagefn()')
+            except Exception as e:
+                if verbose: pymupdf.log(f'{os.getpid()=}: exception {e=}')
+                ret = e
+            if verbose:
+                pymupdf.log(f'{os.getpid()=}: sending {page_num=} {ret=}')
+                
+            queue_up.put( (page_num, ret) )
+
+    error = None
+
+    pids = list()
+    try:
+        # Start child processes.
+        if stats:
+            t = time.time()
+        for i in range(concurrency):
+            p = os.fork()   # pylint: disable=no-member
+            if p == 0:
+                # Child process.
+                try:
+                    try:
+                        childfn()
+                    except Exception as e:
+                        pymupdf.log(f'{os.getpid()=}: childfn() => {e=}')
+                        raise
+                finally:
+                    if verbose:
+                        pymupdf.log(f'{os.getpid()=}: calling os._exit(0)')
+                    os._exit(0)
+            pids.append(p)
+        if stats:
+            _stats_write(t, 'create child processes')
+
+        # Send page numbers.
+        if stats:
+            t = time.time()
+        if verbose:
+            pymupdf.log(f'Sending page numbers.')
+        for page_num in range(len(pages)):
+            queue_down.put(page_num)
+        if stats:
+            _stats_write(t, 'Send page numbers')
+
+        # Collect results. We give up if any worker sends an exception instead
+        # of text, but this hasn't been tested.
+        ret = [None] * len(pages)
+        for i in range(len(pages)):
+            page_num, text = queue_up.get()
+            if verbose:
+                pymupdf.log(f'{page_num=} {len(text)=}')
+            assert ret[page_num] is None
+            if isinstance(text, Exception):
+                if not error:
+                    error = text
+                break
+            ret[page_num] = text
+
+        # Close queue. This should cause exception in workers and terminate
+        # them, but on macos-arm64 this does not seem to happen, so we also
+        # send None, which makes workers terminate.
+        for i in range(concurrency):
+            queue_down.put(None)
+        if verbose: pymupdf.log(f'Closing queues.')
+        queue_down.close()
+
+        if error:
+            raise error
+        if verbose:
+            pymupdf.log(f'After concurrent, returning {len(ret)=}')
+        return ret
+        
+    finally:
+        # Join all child processes.
+        if stats:
+            t = time.time()
+        for pid in pids:
+            if verbose:
+                pymupdf.log(f'waiting for {pid=}.')
+            e = os.waitpid(pid, 0)
+            if verbose:
+                pymupdf.log(f'{pid=} => {e=}')
+        if stats:
+            _stats_write(t, 'Join all child proceses')
diff --git a/lib/python3.1/site-packages/pymupdf/_build.py b/lib/python3.1/site-packages/pymupdf/_build.py
new file mode 100644
index 0000000000000000000000000000000000000000..51c420021316497bafb77964d1b292622b191d71
--- /dev/null
+++ b/lib/python3.1/site-packages/pymupdf/_build.py
@@ -0,0 +1 @@
+mupdf_location=''
diff --git a/lib/python3.1/site-packages/pymupdf/_wxcolors.py b/lib/python3.1/site-packages/pymupdf/_wxcolors.py
new file mode 100644
index 0000000000000000000000000000000000000000..730fcbc3bae4c23fe9fd4fbf3de291d359a835df
--- /dev/null
+++ b/lib/python3.1/site-packages/pymupdf/_wxcolors.py
@@ -0,0 +1,562 @@
+_wxcolors = [
+        ("ALICEBLUE", 240, 248, 255),
+        ("ANTIQUEWHITE", 250, 235, 215),
+        ("ANTIQUEWHITE1", 255, 239, 219),
+        ("ANTIQUEWHITE2", 238, 223, 204),
+        ("ANTIQUEWHITE3", 205, 192, 176),
+        ("ANTIQUEWHITE4", 139, 131, 120),
+        ("AQUA", 0, 255, 255),
+        ("AQUAMARINE", 127, 255, 212),
+        ("AQUAMARINE1", 127, 255, 212),
+        ("AQUAMARINE2", 118, 238, 198),
+        ("AQUAMARINE3", 102, 205, 170),
+        ("AQUAMARINE4", 69, 139, 116),
+        ("AZURE", 240, 255, 255),
+        ("AZURE1", 240, 255, 255),
+        ("AZURE2", 224, 238, 238),
+        ("AZURE3", 193, 205, 205),
+        ("AZURE4", 131, 139, 139),
+        ("BEIGE", 245, 245, 220),
+        ("BISQUE", 255, 228, 196),
+        ("BISQUE1", 255, 228, 196),
+        ("BISQUE2", 238, 213, 183),
+        ("BISQUE3", 205, 183, 158),
+        ("BISQUE4", 139, 125, 107),
+        ("BLACK", 0, 0, 0),
+        ("BLANCHEDALMOND", 255, 235, 205),
+        ("BLUE", 0, 0, 255),
+        ("BLUE1", 0, 0, 255),
+        ("BLUE2", 0, 0, 238),
+        ("BLUE3", 0, 0, 205),
+        ("BLUE4", 0, 0, 139),
+        ("BLUEVIOLET", 138, 43, 226),
+        ("BROWN", 165, 42, 42),
+        ("BROWN1", 255, 64, 64),
+        ("BROWN2", 238, 59, 59),
+        ("BROWN3", 205, 51, 51),
+        ("BROWN4", 139, 35, 35),
+        ("BURLYWOOD", 222, 184, 135),
+        ("BURLYWOOD1", 255, 211, 155),
+        ("BURLYWOOD2", 238, 197, 145),
+        ("BURLYWOOD3", 205, 170, 125),
+        ("BURLYWOOD4", 139, 115, 85),
+        ("CADETBLUE", 95, 158, 160),
+        ("CADETBLUE1", 152, 245, 255),
+        ("CADETBLUE2", 142, 229, 238),
+        ("CADETBLUE3", 122, 197, 205),
+        ("CADETBLUE4", 83, 134, 139),
+        ("CHARTREUSE", 127, 255, 0),
+        ("CHARTREUSE1", 127, 255, 0),
+        ("CHARTREUSE2", 118, 238, 0),
+        ("CHARTREUSE3", 102, 205, 0),
+        ("CHARTREUSE4", 69, 139, 0),
+        ("CHOCOLATE", 210, 105, 30),
+        ("CHOCOLATE1", 255, 127, 36),
+        ("CHOCOLATE2", 238, 118, 33),
+        ("CHOCOLATE3", 205, 102, 29),
+        ("CHOCOLATE4", 139, 69, 19),
+        ("COFFEE", 156, 79, 0),
+        ("CORAL", 255, 127, 80),
+        ("CORAL1", 255, 114, 86),
+        ("CORAL2", 238, 106, 80),
+        ("CORAL3", 205, 91, 69),
+        ("CORAL4", 139, 62, 47),
+        ("CORNFLOWERBLUE", 100, 149, 237),
+        ("CORNSILK", 255, 248, 220),
+        ("CORNSILK1", 255, 248, 220),
+        ("CORNSILK2", 238, 232, 205),
+        ("CORNSILK3", 205, 200, 177),
+        ("CORNSILK4", 139, 136, 120),
+        ("CRIMSON", 220, 20, 60),
+        ("CYAN", 0, 255, 255),
+        ("CYAN1", 0, 255, 255),
+        ("CYAN2", 0, 238, 238),
+        ("CYAN3", 0, 205, 205),
+        ("CYAN4", 0, 139, 139),
+        ("DARKBLUE", 0, 0, 139),
+        ("DARKCYAN", 0, 139, 139),
+        ("DARKGOLDENROD", 184, 134, 11),
+        ("DARKGOLDENROD1", 255, 185, 15),
+        ("DARKGOLDENROD2", 238, 173, 14),
+        ("DARKGOLDENROD3", 205, 149, 12),
+        ("DARKGOLDENROD4", 139, 101, 8),
+        ("DARKGRAY", 169, 169, 169),
+        ("DARKGREEN", 0, 100, 0),
+        ("DARKGREY", 169, 169, 169),
+        ("DARKKHAKI", 189, 183, 107),
+        ("DARKMAGENTA", 139, 0, 139),
+        ("DARKOLIVEGREEN", 85, 107, 47),
+        ("DARKOLIVEGREEN1", 202, 255, 112),
+        ("DARKOLIVEGREEN2", 188, 238, 104),
+        ("DARKOLIVEGREEN3", 162, 205, 90),
+        ("DARKOLIVEGREEN4", 110, 139, 61),
+        ("DARKORANGE", 255, 140, 0),
+        ("DARKORANGE1", 255, 127, 0),
+        ("DARKORANGE2", 238, 118, 0),
+        ("DARKORANGE3", 205, 102, 0),
+        ("DARKORANGE4", 139, 69, 0),
+        ("DARKORCHID", 153, 50, 204),
+        ("DARKORCHID1", 191, 62, 255),
+        ("DARKORCHID2", 178, 58, 238),
+        ("DARKORCHID3", 154, 50, 205),
+        ("DARKORCHID4", 104, 34, 139),
+        ("DARKRED", 139, 0, 0),
+        ("DARKSALMON", 233, 150, 122),
+        ("DARKSEAGREEN", 143, 188, 143),
+        ("DARKSEAGREEN1", 193, 255, 193),
+        ("DARKSEAGREEN2", 180, 238, 180),
+        ("DARKSEAGREEN3", 155, 205, 155),
+        ("DARKSEAGREEN4", 105, 139, 105),
+        ("DARKSLATEBLUE", 72, 61, 139),
+        ("DARKSLATEGRAY", 47, 79, 79),
+        ("DARKSLATEGREY", 47, 79, 79),
+        ("DARKTURQUOISE", 0, 206, 209),
+        ("DARKVIOLET", 148, 0, 211),
+        ("DEEPPINK", 255, 20, 147),
+        ("DEEPPINK1", 255, 20, 147),
+        ("DEEPPINK2", 238, 18, 137),
+        ("DEEPPINK3", 205, 16, 118),
+        ("DEEPPINK4", 139, 10, 80),
+        ("DEEPSKYBLUE", 0, 191, 255),
+        ("DEEPSKYBLUE1", 0, 191, 255),
+        ("DEEPSKYBLUE2", 0, 178, 238),
+        ("DEEPSKYBLUE3", 0, 154, 205),
+        ("DEEPSKYBLUE4", 0, 104, 139),
+        ("DIMGRAY", 105, 105, 105),
+        ("DIMGREY", 105, 105, 105),
+        ("DODGERBLUE", 30, 144, 255),
+        ("DODGERBLUE1", 30, 144, 255),
+        ("DODGERBLUE2", 28, 134, 238),
+        ("DODGERBLUE3", 24, 116, 205),
+        ("DODGERBLUE4", 16, 78, 139),
+        ("FIREBRICK", 178, 34, 34),
+        ("FIREBRICK1", 255, 48, 48),
+        ("FIREBRICK2", 238, 44, 44),
+        ("FIREBRICK3", 205, 38, 38),
+        ("FIREBRICK4", 139, 26, 26),
+        ("FLORALWHITE", 255, 250, 240),
+        ("FORESTGREEN", 34, 139, 34),
+        ("FUCHSIA", 255, 0, 255),
+        ("GAINSBORO", 220, 220, 220),
+        ("GHOSTWHITE", 248, 248, 255),
+        ("GOLD", 255, 215, 0),
+        ("GOLD1", 255, 215, 0),
+        ("GOLD2", 238, 201, 0),
+        ("GOLD3", 205, 173, 0),
+        ("GOLD4", 139, 117, 0),
+        ("GOLDENROD", 218, 165, 32),
+        ("GOLDENROD1", 255, 193, 37),
+        ("GOLDENROD2", 238, 180, 34),
+        ("GOLDENROD3", 205, 155, 29),
+        ("GOLDENROD4", 139, 105, 20),
+        ("GRAY", 190, 190, 190),
+        ("GRAY0", 0, 0, 0),
+        ("GRAY1", 3, 3, 3),
+        ("GRAY10", 26, 26, 26),
+        ("GRAY100", 255, 255, 255),
+        ("GRAY11", 28, 28, 28),
+        ("GRAY12", 31, 31, 31),
+        ("GRAY13", 33, 33, 33),
+        ("GRAY14", 36, 36, 36),
+        ("GRAY15", 38, 38, 38),
+        ("GRAY16", 41, 41, 41),
+        ("GRAY17", 43, 43, 43),
+        ("GRAY18", 46, 46, 46),
+        ("GRAY19", 48, 48, 48),
+        ("GRAY2", 5, 5, 5),
+        ("GRAY20", 51, 51, 51),
+        ("GRAY21", 54, 54, 54),
+        ("GRAY22", 56, 56, 56),
+        ("GRAY23", 59, 59, 59),
+        ("GRAY24", 61, 61, 61),
+        ("GRAY25", 64, 64, 64),
+        ("GRAY26", 66, 66, 66),
+        ("GRAY27", 69, 69, 69),
+        ("GRAY28", 71, 71, 71),
+        ("GRAY29", 74, 74, 74),
+        ("GRAY3", 8, 8, 8),
+        ("GRAY30", 77, 77, 77),
+        ("GRAY31", 79, 79, 79),
+        ("GRAY32", 82, 82, 82),
+        ("GRAY33", 84, 84, 84),
+        ("GRAY34", 87, 87, 87),
+        ("GRAY35", 89, 89, 89),
+        ("GRAY36", 92, 92, 92),
+        ("GRAY37", 94, 94, 94),
+        ("GRAY38", 97, 97, 97),
+        ("GRAY39", 99, 99, 99),
+        ("GRAY4", 10, 10, 10),
+        ("GRAY40", 102, 102, 102),
+        ("GRAY41", 105, 105, 105),
+        ("GRAY42", 107, 107, 107),
+        ("GRAY43", 110, 110, 110),
+        ("GRAY44", 112, 112, 112),
+        ("GRAY45", 115, 115, 115),
+        ("GRAY46", 117, 117, 117),
+        ("GRAY47", 120, 120, 120),
+        ("GRAY48", 122, 122, 122),
+        ("GRAY49", 125, 125, 125),
+        ("GRAY5", 13, 13, 13),
+        ("GRAY50", 127, 127, 127),
+        ("GRAY51", 130, 130, 130),
+        ("GRAY52", 133, 133, 133),
+        ("GRAY53", 135, 135, 135),
+        ("GRAY54", 138, 138, 138),
+        ("GRAY55", 140, 140, 140),
+        ("GRAY56", 143, 143, 143),
+        ("GRAY57", 145, 145, 145),
+        ("GRAY58", 148, 148, 148),
+        ("GRAY59", 150, 150, 150),
+        ("GRAY6", 15, 15, 15),
+        ("GRAY60", 153, 153, 153),
+        ("GRAY61", 156, 156, 156),
+        ("GRAY62", 158, 158, 158),
+        ("GRAY63", 161, 161, 161),
+        ("GRAY64", 163, 163, 163),
+        ("GRAY65", 166, 166, 166),
+        ("GRAY66", 168, 168, 168),
+        ("GRAY67", 171, 171, 171),
+        ("GRAY68", 173, 173, 173),
+        ("GRAY69", 176, 176, 176),
+        ("GRAY7", 18, 18, 18),
+        ("GRAY70", 179, 179, 179),
+        ("GRAY71", 181, 181, 181),
+        ("GRAY72", 184, 184, 184),
+        ("GRAY73", 186, 186, 186),
+        ("GRAY74", 189, 189, 189),
+        ("GRAY75", 191, 191, 191),
+        ("GRAY76", 194, 194, 194),
+        ("GRAY77", 196, 196, 196),
+        ("GRAY78", 199, 199, 199),
+        ("GRAY79", 201, 201, 201),
+        ("GRAY8", 20, 20, 20),
+        ("GRAY80", 204, 204, 204),
+        ("GRAY81", 207, 207, 207),
+        ("GRAY82", 209, 209, 209),
+        ("GRAY83", 212, 212, 212),
+        ("GRAY84", 214, 214, 214),
+        ("GRAY85", 217, 217, 217),
+        ("GRAY86", 219, 219, 219),
+        ("GRAY87", 222, 222, 222),
+        ("GRAY88", 224, 224, 224),
+        ("GRAY89", 227, 227, 227),
+        ("GRAY9", 23, 23, 23),
+        ("GRAY90", 229, 229, 229),
+        ("GRAY91", 232, 232, 232),
+        ("GRAY92", 235, 235, 235),
+        ("GRAY93", 237, 237, 237),
+        ("GRAY94", 240, 240, 240),
+        ("GRAY95", 242, 242, 242),
+        ("GRAY96", 245, 245, 245),
+        ("GRAY97", 247, 247, 247),
+        ("GRAY98", 250, 250, 250),
+        ("GRAY99", 252, 252, 252),
+        ("GREEN YELLOW", 173, 255, 47),
+        ("GREEN", 0, 255, 0),
+        ("GREEN1", 0, 255, 0),
+        ("GREEN2", 0, 238, 0),
+        ("GREEN3", 0, 205, 0),
+        ("GREEN4", 0, 139, 0),
+        ("GREENYELLOW", 173, 255, 47),
+        ("GREY", 128, 128, 128),
+        ("HONEYDEW", 240, 255, 240),
+        ("HONEYDEW1", 240, 255, 240),
+        ("HONEYDEW2", 224, 238, 224),
+        ("HONEYDEW3", 193, 205, 193),
+        ("HONEYDEW4", 131, 139, 131),
+        ("HOTPINK", 255, 105, 180),
+        ("HOTPINK1", 255, 110, 180),
+        ("HOTPINK2", 238, 106, 167),
+        ("HOTPINK3", 205, 96, 144),
+        ("HOTPINK4", 139, 58, 98),
+        ("INDIANRED", 205, 92, 92),
+        ("INDIANRED1", 255, 106, 106),
+        ("INDIANRED2", 238, 99, 99),
+        ("INDIANRED3", 205, 85, 85),
+        ("INDIANRED4", 139, 58, 58),
+        ("INDIGO", 75, 0, 130),
+        ("IVORY", 255, 255, 240),
+        ("IVORY1", 255, 255, 240),
+        ("IVORY2", 238, 238, 224),
+        ("IVORY3", 205, 205, 193),
+        ("IVORY4", 139, 139, 131),
+        ("KHAKI", 240, 230, 140),
+        ("KHAKI1", 255, 246, 143),
+        ("KHAKI2", 238, 230, 133),
+        ("KHAKI3", 205, 198, 115),
+        ("KHAKI4", 139, 134, 78),
+        ("LAVENDER", 230, 230, 250),
+        ("LAVENDERBLUSH", 255, 240, 245),
+        ("LAVENDERBLUSH1", 255, 240, 245),
+        ("LAVENDERBLUSH2", 238, 224, 229),
+        ("LAVENDERBLUSH3", 205, 193, 197),
+        ("LAVENDERBLUSH4", 139, 131, 134),
+        ("LAWNGREEN", 124, 252, 0),
+        ("LEMONCHIFFON", 255, 250, 205),
+        ("LEMONCHIFFON1", 255, 250, 205),
+        ("LEMONCHIFFON2", 238, 233, 191),
+        ("LEMONCHIFFON3", 205, 201, 165),
+        ("LEMONCHIFFON4", 139, 137, 112),
+        ("LIGHTBLUE", 173, 216, 230),
+        ("LIGHTBLUE1", 191, 239, 255),
+        ("LIGHTBLUE2", 178, 223, 238),
+        ("LIGHTBLUE3", 154, 192, 205),
+        ("LIGHTBLUE4", 104, 131, 139),
+        ("LIGHTCORAL", 240, 128, 128),
+        ("LIGHTCYAN", 224, 255, 255),
+        ("LIGHTCYAN1", 224, 255, 255),
+        ("LIGHTCYAN2", 209, 238, 238),
+        ("LIGHTCYAN3", 180, 205, 205),
+        ("LIGHTCYAN4", 122, 139, 139),
+        ("LIGHTGOLDENROD", 238, 221, 130),
+        ("LIGHTGOLDENROD1", 255, 236, 139),
+        ("LIGHTGOLDENROD2", 238, 220, 130),
+        ("LIGHTGOLDENROD3", 205, 190, 112),
+        ("LIGHTGOLDENROD4", 139, 129, 76),
+        ("LIGHTGOLDENRODYELLOW", 250, 250, 210),
+        ("LIGHTGRAY", 211, 211, 211),
+        ("LIGHTGREEN", 144, 238, 144),
+        ("LIGHTGREY", 211, 211, 211),
+        ("LIGHTPINK", 255, 182, 193),
+        ("LIGHTPINK1", 255, 174, 185),
+        ("LIGHTPINK2", 238, 162, 173),
+        ("LIGHTPINK3", 205, 140, 149),
+        ("LIGHTPINK4", 139, 95, 101),
+        ("LIGHTSALMON", 255, 160, 122),
+        ("LIGHTSALMON1", 255, 160, 122),
+        ("LIGHTSALMON2", 238, 149, 114),
+        ("LIGHTSALMON3", 205, 129, 98),
+        ("LIGHTSALMON4", 139, 87, 66),
+        ("LIGHTSEAGREEN", 32, 178, 170),
+        ("LIGHTSKYBLUE", 135, 206, 250),
+        ("LIGHTSKYBLUE1", 176, 226, 255),
+        ("LIGHTSKYBLUE2", 164, 211, 238),
+        ("LIGHTSKYBLUE3", 141, 182, 205),
+        ("LIGHTSKYBLUE4", 96, 123, 139),
+        ("LIGHTSLATEBLUE", 132, 112, 255),
+        ("LIGHTSLATEGRAY", 119, 136, 153),
+        ("LIGHTSLATEGREY", 119, 136, 153),
+        ("LIGHTSTEELBLUE", 176, 196, 222),
+        ("LIGHTSTEELBLUE1", 202, 225, 255),
+        ("LIGHTSTEELBLUE2", 188, 210, 238),
+        ("LIGHTSTEELBLUE3", 162, 181, 205),
+        ("LIGHTSTEELBLUE4", 110, 123, 139),
+        ("LIGHTYELLOW", 255, 255, 224),
+        ("LIGHTYELLOW1", 255, 255, 224),
+        ("LIGHTYELLOW2", 238, 238, 209),
+        ("LIGHTYELLOW3", 205, 205, 180),
+        ("LIGHTYELLOW4", 139, 139, 122),
+        ("LIME", 0, 255, 0),
+        ("LIMEGREEN", 50, 205, 50),
+        ("LINEN", 250, 240, 230),
+        ("MAGENTA", 255, 0, 255),
+        ("MAGENTA1", 255, 0, 255),
+        ("MAGENTA2", 238, 0, 238),
+        ("MAGENTA3", 205, 0, 205),
+        ("MAGENTA4", 139, 0, 139),
+        ("MAROON", 176, 48, 96),
+        ("MAROON1", 255, 52, 179),
+        ("MAROON2", 238, 48, 167),
+        ("MAROON3", 205, 41, 144),
+        ("MAROON4", 139, 28, 98),
+        ("MEDIUMAQUAMARINE", 102, 205, 170),
+        ("MEDIUMBLUE", 0, 0, 205),
+        ("MEDIUMORCHID", 186, 85, 211),
+        ("MEDIUMORCHID1", 224, 102, 255),
+        ("MEDIUMORCHID2", 209, 95, 238),
+        ("MEDIUMORCHID3", 180, 82, 205),
+        ("MEDIUMORCHID4", 122, 55, 139),
+        ("MEDIUMPURPLE", 147, 112, 219),
+        ("MEDIUMPURPLE1", 171, 130, 255),
+        ("MEDIUMPURPLE2", 159, 121, 238),
+        ("MEDIUMPURPLE3", 137, 104, 205),
+        ("MEDIUMPURPLE4", 93, 71, 139),
+        ("MEDIUMSEAGREEN", 60, 179, 113),
+        ("MEDIUMSLATEBLUE", 123, 104, 238),
+        ("MEDIUMSPRINGGREEN", 0, 250, 154),
+        ("MEDIUMTURQUOISE", 72, 209, 204),
+        ("MEDIUMVIOLETRED", 199, 21, 133),
+        ("MIDNIGHTBLUE", 25, 25, 112),
+        ("MINTCREAM", 245, 255, 250),
+        ("MISTYROSE", 255, 228, 225),
+        ("MISTYROSE1", 255, 228, 225),
+        ("MISTYROSE2", 238, 213, 210),
+        ("MISTYROSE3", 205, 183, 181),
+        ("MISTYROSE4", 139, 125, 123),
+        ("MOCCASIN", 255, 228, 181),
+        ("MUPDFBLUE", 37, 114, 172),
+        ("NAVAJOWHITE", 255, 222, 173),
+        ("NAVAJOWHITE1", 255, 222, 173),
+        ("NAVAJOWHITE2", 238, 207, 161),
+        ("NAVAJOWHITE3", 205, 179, 139),
+        ("NAVAJOWHITE4", 139, 121, 94),
+        ("NAVY", 0, 0, 128),
+        ("NAVYBLUE", 0, 0, 128),
+        ("OLDLACE", 253, 245, 230),
+        ("OLIVE", 128, 128, 0),
+        ("OLIVEDRAB", 107, 142, 35),
+        ("OLIVEDRAB1", 192, 255, 62),
+        ("OLIVEDRAB2", 179, 238, 58),
+        ("OLIVEDRAB3", 154, 205, 50),
+        ("OLIVEDRAB4", 105, 139, 34),
+        ("ORANGE", 255, 165, 0),
+        ("ORANGE1", 255, 165, 0),
+        ("ORANGE2", 238, 154, 0),
+        ("ORANGE3", 205, 133, 0),
+        ("ORANGE4", 139, 90, 0),
+        ("ORANGERED", 255, 69, 0),
+        ("ORANGERED1", 255, 69, 0),
+        ("ORANGERED2", 238, 64, 0),
+        ("ORANGERED3", 205, 55, 0),
+        ("ORANGERED4", 139, 37, 0),
+        ("ORCHID", 218, 112, 214),
+        ("ORCHID1", 255, 131, 250),
+        ("ORCHID2", 238, 122, 233),
+        ("ORCHID3", 205, 105, 201),
+        ("ORCHID4", 139, 71, 137),
+        ("PALEGOLDENROD", 238, 232, 170),
+        ("PALEGREEN", 152, 251, 152),
+        ("PALEGREEN1", 154, 255, 154),
+        ("PALEGREEN2", 144, 238, 144),
+        ("PALEGREEN3", 124, 205, 124),
+        ("PALEGREEN4", 84, 139, 84),
+        ("PALETURQUOISE", 175, 238, 238),
+        ("PALETURQUOISE1", 187, 255, 255),
+        ("PALETURQUOISE2", 174, 238, 238),
+        ("PALETURQUOISE3", 150, 205, 205),
+        ("PALETURQUOISE4", 102, 139, 139),
+        ("PALEVIOLETRED", 219, 112, 147),
+        ("PALEVIOLETRED1", 255, 130, 171),
+        ("PALEVIOLETRED2", 238, 121, 159),
+        ("PALEVIOLETRED3", 205, 104, 137),
+        ("PALEVIOLETRED4", 139, 71, 93),
+        ("PAPAYAWHIP", 255, 239, 213),
+        ("PEACHPUFF", 255, 218, 185),
+        ("PEACHPUFF1", 255, 218, 185),
+        ("PEACHPUFF2", 238, 203, 173),
+        ("PEACHPUFF3", 205, 175, 149),
+        ("PEACHPUFF4", 139, 119, 101),
+        ("PERU", 205, 133, 63),
+        ("PINK", 255, 192, 203),
+        ("PINK1", 255, 181, 197),
+        ("PINK2", 238, 169, 184),
+        ("PINK3", 205, 145, 158),
+        ("PINK4", 139, 99, 108),
+        ("PLUM", 221, 160, 221),
+        ("PLUM1", 255, 187, 255),
+        ("PLUM2", 238, 174, 238),
+        ("PLUM3", 205, 150, 205),
+        ("PLUM4", 139, 102, 139),
+        ("POWDERBLUE", 176, 224, 230),
+        ("PURPLE", 160, 32, 240),
+        ("PURPLE1", 155, 48, 255),
+        ("PURPLE2", 145, 44, 238),
+        ("PURPLE3", 125, 38, 205),
+        ("PURPLE4", 85, 26, 139),
+        ("PY_COLOR", 240, 255, 210),
+        ("RED", 255, 0, 0),
+        ("RED1", 255, 0, 0),
+        ("RED2", 238, 0, 0),
+        ("RED3", 205, 0, 0),
+        ("RED4", 139, 0, 0),
+        ("ROSYBROWN", 188, 143, 143),
+        ("ROSYBROWN1", 255, 193, 193),
+        ("ROSYBROWN2", 238, 180, 180),
+        ("ROSYBROWN3", 205, 155, 155),
+        ("ROSYBROWN4", 139, 105, 105),
+        ("ROYALBLUE", 65, 105, 225),
+        ("ROYALBLUE1", 72, 118, 255),
+        ("ROYALBLUE2", 67, 110, 238),
+        ("ROYALBLUE3", 58, 95, 205),
+        ("ROYALBLUE4", 39, 64, 139),
+        ("SADDLEBROWN", 139, 69, 19),
+        ("SALMON", 250, 128, 114),
+        ("SALMON1", 255, 140, 105),
+        ("SALMON2", 238, 130, 98),
+        ("SALMON3", 205, 112, 84),
+        ("SALMON4", 139, 76, 57),
+        ("SANDYBROWN", 244, 164, 96),
+        ("SEAGREEN", 46, 139, 87),
+        ("SEAGREEN1", 84, 255, 159),
+        ("SEAGREEN2", 78, 238, 148),
+        ("SEAGREEN3", 67, 205, 128),
+        ("SEAGREEN4", 46, 139, 87),
+        ("SEASHELL", 255, 245, 238),
+        ("SEASHELL1", 255, 245, 238),
+        ("SEASHELL2", 238, 229, 222),
+        ("SEASHELL3", 205, 197, 191),
+        ("SEASHELL4", 139, 134, 130),
+        ("SIENNA", 160, 82, 45),
+        ("SIENNA1", 255, 130, 71),
+        ("SIENNA2", 238, 121, 66),
+        ("SIENNA3", 205, 104, 57),
+        ("SIENNA4", 139, 71, 38),
+        ("SILVER", 192, 192, 192),
+        ("SKYBLUE", 135, 206, 235),
+        ("SKYBLUE1", 135, 206, 255),
+        ("SKYBLUE2", 126, 192, 238),
+        ("SKYBLUE3", 108, 166, 205),
+        ("SKYBLUE4", 74, 112, 139),
+        ("SLATEBLUE", 106, 90, 205),
+        ("SLATEBLUE1", 131, 111, 255),
+        ("SLATEBLUE2", 122, 103, 238),
+        ("SLATEBLUE3", 105, 89, 205),
+        ("SLATEBLUE4", 71, 60, 139),
+        ("SLATEGRAY", 112, 128, 144),
+        ("SLATEGREY", 112, 128, 144),
+        ("SNOW", 255, 250, 250),
+        ("SNOW1", 255, 250, 250),
+        ("SNOW2", 238, 233, 233),
+        ("SNOW3", 205, 201, 201),
+        ("SNOW4", 139, 137, 137),
+        ("SPRINGGREEN", 0, 255, 127),
+        ("SPRINGGREEN1", 0, 255, 127),
+        ("SPRINGGREEN2", 0, 238, 118),
+        ("SPRINGGREEN3", 0, 205, 102),
+        ("SPRINGGREEN4", 0, 139, 69),
+        ("STEELBLUE", 70, 130, 180),
+        ("STEELBLUE1", 99, 184, 255),
+        ("STEELBLUE2", 92, 172, 238),
+        ("STEELBLUE3", 79, 148, 205),
+        ("STEELBLUE4", 54, 100, 139),
+        ("TAN", 210, 180, 140),
+        ("TAN1", 255, 165, 79),
+        ("TAN2", 238, 154, 73),
+        ("TAN3", 205, 133, 63),
+        ("TAN4", 139, 90, 43),
+        ("TEAL", 0, 128, 128),
+        ("THISTLE", 216, 191, 216),
+        ("THISTLE1", 255, 225, 255),
+        ("THISTLE2", 238, 210, 238),
+        ("THISTLE3", 205, 181, 205),
+        ("THISTLE4", 139, 123, 139),
+        ("TOMATO", 255, 99, 71),
+        ("TOMATO1", 255, 99, 71),
+        ("TOMATO2", 238, 92, 66),
+        ("TOMATO3", 205, 79, 57),
+        ("TOMATO4", 139, 54, 38),
+        ("TURQUOISE", 64, 224, 208),
+        ("TURQUOISE1", 0, 245, 255),
+        ("TURQUOISE2", 0, 229, 238),
+        ("TURQUOISE3", 0, 197, 205),
+        ("TURQUOISE4", 0, 134, 139),
+        ("VIOLET", 238, 130, 238),
+        ("VIOLETRED", 208, 32, 144),
+        ("VIOLETRED1", 255, 62, 150),
+        ("VIOLETRED2", 238, 58, 140),
+        ("VIOLETRED3", 205, 50, 120),
+        ("VIOLETRED4", 139, 34, 82),
+        ("WHEAT", 245, 222, 179),
+        ("WHEAT1", 255, 231, 186),
+        ("WHEAT2", 238, 216, 174),
+        ("WHEAT3", 205, 186, 150),
+        ("WHEAT4", 139, 126, 102),
+        ("WHITE", 255, 255, 255),
+        ("WHITESMOKE", 245, 245, 245),
+        ("YELLOW", 255, 255, 0),
+        ("YELLOW1", 255, 255, 0),
+        ("YELLOW2", 238, 238, 0),
+        ("YELLOW3", 205, 205, 0),
+        ("YELLOW4", 139, 139, 0),
+        ("YELLOWGREEN", 154, 205, 50),
+        ]
diff --git a/lib/python3.1/site-packages/pymupdf/extra.py b/lib/python3.1/site-packages/pymupdf/extra.py
new file mode 100644
index 0000000000000000000000000000000000000000..633da4d72630565e9c877fb676cdff28358cbb20
--- /dev/null
+++ b/lib/python3.1/site-packages/pymupdf/extra.py
@@ -0,0 +1,217 @@
+# This file was automatically generated by SWIG (https://www.swig.org).
+# Version 4.3.1
+#
+# Do not make changes to this file unless you know what you are doing - modify
+# the SWIG interface file instead.
+
+from sys import version_info as _swig_python_version_info
+# Import the low-level C/C++ module
+if __package__ or "." in __name__:
+    from . import _extra
+else:
+    import _extra
+
+try:
+    import builtins as __builtin__
+except ImportError:
+    import __builtin__
+
+def _swig_repr(self):
+    try:
+        strthis = "proxy of " + self.this.__repr__()
+    except __builtin__.Exception:
+        strthis = ""
+    return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,)
+
+
+def _swig_setattr_nondynamic_instance_variable(set):
+    def set_instance_attr(self, name, value):
+        if name == "this":
+            set(self, name, value)
+        elif name == "thisown":
+            self.this.own(value)
+        elif hasattr(self, name) and isinstance(getattr(type(self), name), property):
+            set(self, name, value)
+        else:
+            raise AttributeError("You cannot add instance attributes to %s" % self)
+    return set_instance_attr
+
+
+def _swig_setattr_nondynamic_class_variable(set):
+    def set_class_attr(cls, name, value):
+        if hasattr(cls, name) and not isinstance(getattr(cls, name), property):
+            set(cls, name, value)
+        else:
+            raise AttributeError("You cannot add class attributes to %s" % cls)
+    return set_class_attr
+
+
+def _swig_add_metaclass(metaclass):
+    """Class decorator for adding a metaclass to a SWIG wrapped class - a slimmed down version of six.add_metaclass"""
+    def wrapper(cls):
+        return metaclass(cls.__name__, cls.__bases__, cls.__dict__.copy())
+    return wrapper
+
+
+class _SwigNonDynamicMeta(type):
+    """Meta class to enforce nondynamic attributes (no new attributes) for a class"""
+    __setattr__ = _swig_setattr_nondynamic_class_variable(type.__setattr__)
+
+
+
+# pylint: disable=all
+
+
+def page_merge(doc_des, doc_src, page_from, page_to, rotate, links, copy_annots, graft_map):
+    return _extra.page_merge(doc_des, doc_src, page_from, page_to, rotate, links, copy_annots, graft_map)
+
+def JM_merge_range(doc_des, doc_src, spage, epage, apage, rotate, links, annots, show_progress, graft_map):
+    return _extra.JM_merge_range(doc_des, doc_src, spage, epage, apage, rotate, links, annots, show_progress, graft_map)
+
+def FzDocument_insert_pdf(doc, src, from_page, to_page, start_at, rotate, links, annots, show_progress, final, graft_map):
+    return _extra.FzDocument_insert_pdf(doc, src, from_page, to_page, start_at, rotate, links, annots, show_progress, final, graft_map)
+
+def page_xref(this_doc, pno):
+    return _extra.page_xref(this_doc, pno)
+
+def _newPage(*args):
+    return _extra._newPage(*args)
+
+def JM_add_annot_id(annot, stem):
+    return _extra.JM_add_annot_id(annot, stem)
+
+def JM_set_annot_callout_line(annot, callout, count):
+    return _extra.JM_set_annot_callout_line(annot, callout, count)
+
+def JM_get_annot_id_list(page):
+    return _extra.JM_get_annot_id_list(page)
+
+def _add_caret_annot(*args):
+    return _extra._add_caret_annot(*args)
+
+def Tools_parse_da(this_annot):
+    return _extra.Tools_parse_da(this_annot)
+
+def Annot_getAP(annot):
+    return _extra.Annot_getAP(annot)
+
+def Tools_update_da(this_annot, da_str):
+    return _extra.Tools_update_da(this_annot, da_str)
+
+def JM_point_from_py(p):
+    return _extra.JM_point_from_py(p)
+
+def Annot_rect(annot):
+    return _extra.Annot_rect(annot)
+
+def util_transform_rect(rect, matrix):
+    return _extra.util_transform_rect(rect, matrix)
+
+def Annot_rect3(annot):
+    return _extra.Annot_rect3(annot)
+
+def Page_derotate_matrix(*args):
+    return _extra.Page_derotate_matrix(*args)
+
+def JM_get_annot_xref_list(page_obj):
+    return _extra.JM_get_annot_xref_list(page_obj)
+
+def xref_object(*args):
+    return _extra.xref_object(*args)
+
+def Link_is_external(this_link):
+    return _extra.Link_is_external(this_link)
+
+def Page_addAnnot_FromString(*args):
+    return _extra.Page_addAnnot_FromString(*args)
+
+def Link_next(this_link):
+    return _extra.Link_next(this_link)
+
+def page_count_fz2(document):
+    return _extra.page_count_fz2(document)
+
+def page_count_fz(document):
+    return _extra.page_count_fz(document)
+
+def page_count_pdf(pdf):
+    return _extra.page_count_pdf(pdf)
+
+def page_count(*args):
+    return _extra.page_count(*args)
+
+def page_annot_xrefs(*args):
+    return _extra.page_annot_xrefs(*args)
+
+def Outline_is_external(outline):
+    return _extra.Outline_is_external(outline)
+
+def Document_extend_toc_items(*args):
+    return _extra.Document_extend_toc_items(*args)
+
+def ll_fz_absi(i):
+    return _extra.ll_fz_absi(i)
+
+def JM_new_texttrace_device(out):
+    return _extra.JM_new_texttrace_device(out)
+
+def JM_char_bbox(line, ch):
+    return _extra.JM_char_bbox(line, ch)
+
+def JM_char_quad(line, ch):
+    return _extra.JM_char_quad(line, ch)
+
+def JM_print_stext_page_as_text(res, page):
+    return _extra.JM_print_stext_page_as_text(res, page)
+
+def set_skip_quad_corrections(on):
+    return _extra.set_skip_quad_corrections(on)
+
+def set_subset_fontnames(on):
+    return _extra.set_subset_fontnames(on)
+
+def set_small_glyph_heights(on):
+    return _extra.set_small_glyph_heights(on)
+
+def JM_cropbox(page_obj):
+    return _extra.JM_cropbox(page_obj)
+
+def get_cdrawings(page, extended=None, callback=None, method=None):
+    return _extra.get_cdrawings(page, extended, callback, method)
+
+def JM_make_spanlist(line_dict, line, raw, buff, tp_rect):
+    return _extra.JM_make_spanlist(line_dict, line, raw, buff, tp_rect)
+
+def extractWORDS(this_tpage, delimiters):
+    return _extra.extractWORDS(this_tpage, delimiters)
+
+def extractBLOCKS(_self):
+    return _extra.extractBLOCKS(_self)
+
+def link_uri(link):
+    return _extra.link_uri(link)
+
+def page_get_textpage(_self, clip, flags, matrix):
+    return _extra.page_get_textpage(_self, clip, flags, matrix)
+
+def JM_make_textpage_dict(tp, page_dict, raw):
+    return _extra.JM_make_textpage_dict(tp, page_dict, raw)
+
+def pixmap_pixel(pm, x, y):
+    return _extra.pixmap_pixel(pm, x, y)
+
+def pixmap_n(pixmap):
+    return _extra.pixmap_n(pixmap)
+
+def JM_search_stext_page(page, needle):
+    return _extra.JM_search_stext_page(page, needle)
+
+def set_pixel(pm, x, y, color):
+    return _extra.set_pixel(pm, x, y, color)
+
+def pixmap_copy(pm, src, n):
+    return _extra.pixmap_copy(pm, src, n)
+
+def ll_JM_color_count(pm, clip):
+    return _extra.ll_JM_color_count(pm, clip)
+
diff --git a/lib/python3.1/site-packages/pymupdf/pymupdf.py b/lib/python3.1/site-packages/pymupdf/pymupdf.py
new file mode 100644
index 0000000000000000000000000000000000000000..82205555fcc5c365dec9f8e7486a9048dc275d37
--- /dev/null
+++ b/lib/python3.1/site-packages/pymupdf/pymupdf.py
@@ -0,0 +1,2 @@
+# pylint: disable=wildcard-import,unused-import
+from . import *
diff --git a/lib/python3.1/site-packages/pymupdf/table.py b/lib/python3.1/site-packages/pymupdf/table.py
new file mode 100644
index 0000000000000000000000000000000000000000..4d95ffe576f58293c69c71959a1921df52018150
--- /dev/null
+++ b/lib/python3.1/site-packages/pymupdf/table.py
@@ -0,0 +1,2440 @@
+"""
+Copyright (C) 2023 Artifex Software, Inc.
+
+This file is part of PyMuPDF.
+
+PyMuPDF is free software: you can redistribute it and/or modify it under the
+terms of the GNU Affero General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
+
+PyMuPDF is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+details.
+
+You should have received a copy of the GNU Affero General Public License
+along with MuPDF. If not, see 
+
+Alternative licensing terms are available from the licensor.
+For commercial licensing, see  or contact
+Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco,
+CA 94129, USA, for further information.
+
+---------------------------------------------------------------------
+Portions of this code have been ported from pdfplumber, see
+https://pypi.org/project/pdfplumber/.
+
+The ported code is under the following MIT license:
+
+---------------------------------------------------------------------
+The MIT License (MIT)
+
+Copyright (c) 2015, Jeremy Singer-Vine
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+---------------------------------------------------------------------
+Also see here: https://github.com/jsvine/pdfplumber/blob/stable/LICENSE.txt
+---------------------------------------------------------------------
+
+The porting mainly pertains to files "table.py" and relevant parts of
+"utils/text.py" within pdfplumber's repository on Github.
+With respect to "text.py", we have removed functions or features that are not
+used by table processing. Examples are:
+
+* the text search function
+* simple text extraction
+* text extraction by lines
+
+Original pdfplumber code does neither detect, nor identify table headers.
+This PyMuPDF port adds respective code to the 'Table' class as method '_get_header'.
+This is implemented as new class TableHeader with the properties:
+* bbox: A tuple for the header's bbox
+* cells: A tuple for each bbox of a column header
+* names: A list of strings with column header text
+* external: A bool indicating whether the header is outside the table cells.
+
+"""
+
+import inspect
+import itertools
+import string
+import html
+from collections.abc import Sequence
+from dataclasses import dataclass
+from operator import itemgetter
+import weakref
+
+# -------------------------------------------------------------------
+# Start of PyMuPDF interface code
+# -------------------------------------------------------------------
+from . import (
+    Rect,
+    Matrix,
+    TEXTFLAGS_TEXT,
+    TEXT_FONT_BOLD,
+    TEXT_FONT_SUPERSCRIPT,
+    TOOLS,
+    EMPTY_RECT,
+    sRGB_to_pdf,
+    Point,
+    message,
+)
+
+EDGES = []  # vector graphics from PyMuPDF
+CHARS = []  # text characters from PyMuPDF
+TEXTPAGE = None
+white_spaces = set(string.whitespace)  # for checking white space only cells
+# -------------------------------------------------------------------
+# End of PyMuPDF interface code
+# -------------------------------------------------------------------
+
+
+class UnsetFloat(float):
+    pass
+
+
+NON_NEGATIVE_SETTINGS = [
+    "snap_tolerance",
+    "snap_x_tolerance",
+    "snap_y_tolerance",
+    "join_tolerance",
+    "join_x_tolerance",
+    "join_y_tolerance",
+    "edge_min_length",
+    "min_words_vertical",
+    "min_words_horizontal",
+    "intersection_tolerance",
+    "intersection_x_tolerance",
+    "intersection_y_tolerance",
+]
+
+
+TABLE_STRATEGIES = ["lines", "lines_strict", "text", "explicit"]
+UNSET = UnsetFloat(0)
+DEFAULT_SNAP_TOLERANCE = 3
+DEFAULT_JOIN_TOLERANCE = 3
+DEFAULT_MIN_WORDS_VERTICAL = 3
+DEFAULT_MIN_WORDS_HORIZONTAL = 1
+DEFAULT_X_TOLERANCE = 3
+DEFAULT_Y_TOLERANCE = 3
+DEFAULT_X_DENSITY = 7.25
+DEFAULT_Y_DENSITY = 13
+bbox_getter = itemgetter("x0", "top", "x1", "bottom")
+
+
+LIGATURES = {
+    "ff": "ff",
+    "ffi": "ffi",
+    "ffl": "ffl",
+    "fi": "fi",
+    "fl": "fl",
+    "st": "st",
+    "ſt": "st",
+}
+
+
+def to_list(collection) -> list:
+    if isinstance(collection, list):
+        return collection
+    elif isinstance(collection, Sequence):
+        return list(collection)
+    elif hasattr(collection, "to_dict"):
+        res = collection.to_dict("records")  # pragma: nocover
+        return res
+    else:
+        return list(collection)
+
+
+class TextMap:
+    """
+    A TextMap maps each unicode character in the text to an individual `char`
+    object (or, in the case of layout-implied whitespace, `None`).
+    """
+
+    def __init__(self, tuples=None) -> None:
+        self.tuples = tuples
+        self.as_string = "".join(map(itemgetter(0), tuples))
+
+    def match_to_dict(
+        self,
+        m,
+        main_group: int = 0,
+        return_groups: bool = True,
+        return_chars: bool = True,
+    ) -> dict:
+        subset = self.tuples[m.start(main_group) : m.end(main_group)]
+        chars = [c for (text, c) in subset if c is not None]
+        x0, top, x1, bottom = objects_to_bbox(chars)
+
+        result = {
+            "text": m.group(main_group),
+            "x0": x0,
+            "top": top,
+            "x1": x1,
+            "bottom": bottom,
+        }
+
+        if return_groups:
+            result["groups"] = m.groups()
+
+        if return_chars:
+            result["chars"] = chars
+
+        return result
+
+
+class WordMap:
+    """
+    A WordMap maps words->chars.
+    """
+
+    def __init__(self, tuples) -> None:
+        self.tuples = tuples
+
+    def to_textmap(
+        self,
+        layout: bool = False,
+        layout_width=0,
+        layout_height=0,
+        layout_width_chars: int = 0,
+        layout_height_chars: int = 0,
+        x_density=DEFAULT_X_DENSITY,
+        y_density=DEFAULT_Y_DENSITY,
+        x_shift=0,
+        y_shift=0,
+        y_tolerance=DEFAULT_Y_TOLERANCE,
+        use_text_flow: bool = False,
+        presorted: bool = False,
+        expand_ligatures: bool = True,
+    ) -> TextMap:
+        """
+        Given a list of (word, chars) tuples (i.e., a WordMap), return a list of
+        (char-text, char) tuples (i.e., a TextMap) that can be used to mimic the
+        structural layout of the text on the page(s), using the following approach:
+
+        - Sort the words by (doctop, x0) if not already sorted.
+
+        - Calculate the initial doctop for the starting page.
+
+        - Cluster the words by doctop (taking `y_tolerance` into account), and
+          iterate through them.
+
+        - For each cluster, calculate the distance between that doctop and the
+          initial doctop, in points, minus `y_shift`. Divide that distance by
+          `y_density` to calculate the minimum number of newlines that should come
+          before this cluster. Append that number of newlines *minus* the number of
+          newlines already appended, with a minimum of one.
+
+        - Then for each cluster, iterate through each word in it. Divide each
+          word's x0, minus `x_shift`, by `x_density` to calculate the minimum
+          number of characters that should come before this cluster.  Append that
+          number of spaces *minus* the number of characters and spaces already
+          appended, with a minimum of one. Then append the word's text.
+
+        - At the termination of each line, add more spaces if necessary to
+          mimic `layout_width`.
+
+        - Finally, add newlines to the end if necessary to mimic to
+          `layout_height`.
+
+        Note: This approach currently works best for horizontal, left-to-right
+        text, but will display all words regardless of orientation. There is room
+        for improvement in better supporting right-to-left text, as well as
+        vertical text.
+        """
+        _textmap = []
+
+        if not len(self.tuples):
+            return TextMap(_textmap)
+
+        expansions = LIGATURES if expand_ligatures else {}
+
+        if layout:
+            if layout_width_chars:
+                if layout_width:
+                    raise ValueError(
+                        "`layout_width` and `layout_width_chars` cannot both be set."
+                    )
+            else:
+                layout_width_chars = int(round(layout_width / x_density))
+
+            if layout_height_chars:
+                if layout_height:
+                    raise ValueError(
+                        "`layout_height` and `layout_height_chars` cannot both be set."
+                    )
+            else:
+                layout_height_chars = int(round(layout_height / y_density))
+
+            blank_line = [(" ", None)] * layout_width_chars
+        else:
+            blank_line = []
+
+        num_newlines = 0
+
+        words_sorted_doctop = (
+            self.tuples
+            if presorted or use_text_flow
+            else sorted(self.tuples, key=lambda x: float(x[0]["doctop"]))
+        )
+
+        first_word = words_sorted_doctop[0][0]
+        doctop_start = first_word["doctop"] - first_word["top"]
+
+        for i, ws in enumerate(
+            cluster_objects(
+                words_sorted_doctop, lambda x: float(x[0]["doctop"]), y_tolerance
+            )
+        ):
+            y_dist = (
+                (ws[0][0]["doctop"] - (doctop_start + y_shift)) / y_density
+                if layout
+                else 0
+            )
+            num_newlines_prepend = max(
+                # At least one newline, unless this iis the first line
+                int(i > 0),
+                # ... or as many as needed to get the imputed "distance" from the top
+                round(y_dist) - num_newlines,
+            )
+
+            for i in range(num_newlines_prepend):
+                if not len(_textmap) or _textmap[-1][0] == "\n":
+                    _textmap += blank_line
+                _textmap.append(("\n", None))
+
+            num_newlines += num_newlines_prepend
+
+            line_len = 0
+
+            line_words_sorted_x0 = (
+                ws
+                if presorted or use_text_flow
+                else sorted(ws, key=lambda x: float(x[0]["x0"]))
+            )
+
+            for word, chars in line_words_sorted_x0:
+                x_dist = (word["x0"] - x_shift) / x_density if layout else 0
+                num_spaces_prepend = max(min(1, line_len), round(x_dist) - line_len)
+                _textmap += [(" ", None)] * num_spaces_prepend
+                line_len += num_spaces_prepend
+
+                for c in chars:
+                    letters = expansions.get(c["text"], c["text"])
+                    for letter in letters:
+                        _textmap.append((letter, c))
+                        line_len += 1
+
+            # Append spaces at end of line
+            if layout:
+                _textmap += [(" ", None)] * (layout_width_chars - line_len)
+
+        # Append blank lines at end of text
+        if layout:
+            num_newlines_append = layout_height_chars - (num_newlines + 1)
+            for i in range(num_newlines_append):
+                if i > 0:
+                    _textmap += blank_line
+                _textmap.append(("\n", None))
+
+            # Remove terminal newline
+            if _textmap[-1] == ("\n", None):
+                _textmap = _textmap[:-1]
+
+        return TextMap(_textmap)
+
+
+class WordExtractor:
+    def __init__(
+        self,
+        x_tolerance=DEFAULT_X_TOLERANCE,
+        y_tolerance=DEFAULT_Y_TOLERANCE,
+        keep_blank_chars: bool = False,
+        use_text_flow=False,
+        horizontal_ltr=True,  # Should words be read left-to-right?
+        vertical_ttb=False,  # Should vertical words be read top-to-bottom?
+        extra_attrs=None,
+        split_at_punctuation=False,
+        expand_ligatures=True,
+    ):
+        self.x_tolerance = x_tolerance
+        self.y_tolerance = y_tolerance
+        self.keep_blank_chars = keep_blank_chars
+        self.use_text_flow = use_text_flow
+        self.horizontal_ltr = horizontal_ltr
+        self.vertical_ttb = vertical_ttb
+        self.extra_attrs = [] if extra_attrs is None else extra_attrs
+
+        # Note: string.punctuation = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'
+        self.split_at_punctuation = (
+            string.punctuation
+            if split_at_punctuation is True
+            else (split_at_punctuation or "")
+        )
+
+        self.expansions = LIGATURES if expand_ligatures else {}
+
+    def merge_chars(self, ordered_chars: list):
+        x0, top, x1, bottom = objects_to_bbox(ordered_chars)
+        doctop_adj = ordered_chars[0]["doctop"] - ordered_chars[0]["top"]
+        upright = ordered_chars[0]["upright"]
+        direction = 1 if (self.horizontal_ltr if upright else self.vertical_ttb) else -1
+
+        matrix = ordered_chars[0]["matrix"]
+
+        rotation = 0
+        if not upright and matrix[1] < 0:
+            ordered_chars = reversed(ordered_chars)
+            rotation = 270
+
+        if matrix[0] < 0 and matrix[3] < 0:
+            rotation = 180
+        elif matrix[1] > 0:
+            rotation = 90
+
+        word = {
+            "text": "".join(
+                self.expansions.get(c["text"], c["text"]) for c in ordered_chars
+            ),
+            "x0": x0,
+            "x1": x1,
+            "top": top,
+            "doctop": top + doctop_adj,
+            "bottom": bottom,
+            "upright": upright,
+            "direction": direction,
+            "rotation": rotation,
+        }
+
+        for key in self.extra_attrs:
+            word[key] = ordered_chars[0][key]
+
+        return word
+
+    def char_begins_new_word(
+        self,
+        prev_char,
+        curr_char,
+    ) -> bool:
+        """This method takes several factors into account to determine if
+        `curr_char` represents the beginning of a new word:
+
+        - Whether the text is "upright" (i.e., non-rotated)
+        - Whether the user has specified that horizontal text runs
+          left-to-right (default) or right-to-left, as represented by
+          self.horizontal_ltr
+        - Whether the user has specified that vertical text the text runs
+          top-to-bottom (default) or bottom-to-top, as represented by
+          self.vertical_ttb
+        - The x0, top, x1, and bottom attributes of prev_char and
+          curr_char
+        - The self.x_tolerance and self.y_tolerance settings. Note: In
+          this case, x/y refer to those directions for non-rotated text.
+          For vertical text, they are flipped. A more accurate terminology
+          might be "*intra*line character distance tolerance" and
+          "*inter*line character distance tolerance"
+
+        An important note: The *intra*line distance is measured from the
+        *end* of the previous character to the *beginning* of the current
+        character, while the *inter*line distance is measured from the
+        *top* of the previous character to the *top* of the next
+        character. The reasons for this are partly repository-historical,
+        and partly logical, as successive text lines' bounding boxes often
+        overlap slightly (and we don't want that overlap to be interpreted
+        as the two lines being the same line).
+
+        The upright-ness of the character determines the attributes to
+        compare, while horizontal_ltr/vertical_ttb determine the direction
+        of the comparison.
+        """
+
+        # Note: Due to the grouping step earlier in the process,
+        # curr_char["upright"] will always equal prev_char["upright"].
+        if curr_char["upright"]:
+            x = self.x_tolerance
+            y = self.y_tolerance
+            ay = prev_char["top"]
+            cy = curr_char["top"]
+            if self.horizontal_ltr:
+                ax = prev_char["x0"]
+                bx = prev_char["x1"]
+                cx = curr_char["x0"]
+            else:
+                ax = -prev_char["x1"]
+                bx = -prev_char["x0"]
+                cx = -curr_char["x1"]
+
+        else:
+            x = self.y_tolerance
+            y = self.x_tolerance
+            ay = prev_char["x0"]
+            cy = curr_char["x0"]
+            if self.vertical_ttb:
+                ax = prev_char["top"]
+                bx = prev_char["bottom"]
+                cx = curr_char["top"]
+            else:
+                ax = -prev_char["bottom"]
+                bx = -prev_char["top"]
+                cx = -curr_char["bottom"]
+
+        return bool(
+            # Intraline test
+            (cx < ax)
+            or (cx > bx + x)
+            # Interline test
+            or (cy > ay + y)
+        )
+
+    def iter_chars_to_words(self, ordered_chars):
+        current_word: list = []
+
+        def start_next_word(new_char=None):
+            nonlocal current_word
+
+            if current_word:
+                yield current_word
+
+            current_word = [] if new_char is None else [new_char]
+
+        for char in ordered_chars:
+            text = char["text"]
+
+            if not self.keep_blank_chars and text.isspace():
+                yield from start_next_word(None)
+
+            elif text in self.split_at_punctuation:
+                yield from start_next_word(char)
+                yield from start_next_word(None)
+
+            elif current_word and self.char_begins_new_word(current_word[-1], char):
+                yield from start_next_word(char)
+
+            else:
+                current_word.append(char)
+
+        # Finally, after all chars processed
+        if current_word:
+            yield current_word
+
+    def iter_sort_chars(self, chars):
+        def upright_key(x) -> int:
+            return -int(x["upright"])
+
+        for upright_cluster in cluster_objects(list(chars), upright_key, 0):
+            upright = upright_cluster[0]["upright"]
+            cluster_key = "doctop" if upright else "x0"
+
+            # Cluster by line
+            subclusters = cluster_objects(
+                upright_cluster, itemgetter(cluster_key), self.y_tolerance
+            )
+
+            for sc in subclusters:
+                # Sort within line
+                sort_key = "x0" if upright else "doctop"
+                to_yield = sorted(sc, key=itemgetter(sort_key))
+
+                # Reverse order if necessary
+                if not (self.horizontal_ltr if upright else self.vertical_ttb):
+                    yield from reversed(to_yield)
+                else:
+                    yield from to_yield
+
+    def iter_extract_tuples(self, chars):
+        ordered_chars = chars if self.use_text_flow else self.iter_sort_chars(chars)
+
+        grouping_key = itemgetter("upright", *self.extra_attrs)
+        grouped_chars = itertools.groupby(ordered_chars, grouping_key)
+
+        for keyvals, char_group in grouped_chars:
+            for word_chars in self.iter_chars_to_words(char_group):
+                yield (self.merge_chars(word_chars), word_chars)
+
+    def extract_wordmap(self, chars) -> WordMap:
+        return WordMap(list(self.iter_extract_tuples(chars)))
+
+    def extract_words(self, chars: list) -> list:
+        words = list(word for word, word_chars in self.iter_extract_tuples(chars))
+        return words
+
+
+def extract_words(chars: list, **kwargs) -> list:
+    return WordExtractor(**kwargs).extract_words(chars)
+
+
+TEXTMAP_KWARGS = inspect.signature(WordMap.to_textmap).parameters.keys()
+WORD_EXTRACTOR_KWARGS = inspect.signature(WordExtractor).parameters.keys()
+
+
+def chars_to_textmap(chars: list, **kwargs) -> TextMap:
+    kwargs.update({"presorted": True})
+
+    extractor = WordExtractor(
+        **{k: kwargs[k] for k in WORD_EXTRACTOR_KWARGS if k in kwargs}
+    )
+    wordmap = extractor.extract_wordmap(chars)
+    textmap = wordmap.to_textmap(
+        **{k: kwargs[k] for k in TEXTMAP_KWARGS if k in kwargs}
+    )
+
+    return textmap
+
+
+def extract_text(chars: list, **kwargs) -> str:
+    chars = to_list(chars)
+    if len(chars) == 0:
+        return ""
+
+    if kwargs.get("layout"):
+        return chars_to_textmap(chars, **kwargs).as_string
+    else:
+        y_tolerance = kwargs.get("y_tolerance", DEFAULT_Y_TOLERANCE)
+        extractor = WordExtractor(
+            **{k: kwargs[k] for k in WORD_EXTRACTOR_KWARGS if k in kwargs}
+        )
+        words = extractor.extract_words(chars)
+        if words:
+            rotation = words[0]["rotation"]  # rotation cannot change within a cell
+        else:
+            rotation = 0
+
+        if rotation == 90:
+            words.sort(key=lambda w: (w["x1"], -w["top"]))
+            lines = " ".join([w["text"] for w in words])
+        elif rotation == 270:
+            words.sort(key=lambda w: (-w["x1"], w["top"]))
+            lines = " ".join([w["text"] for w in words])
+        else:
+            lines = cluster_objects(words, itemgetter("doctop"), y_tolerance)
+            lines = "\n".join(" ".join(word["text"] for word in line) for line in lines)
+            if rotation == 180:  # needs extra treatment
+                lines = "".join([(c if c != "\n" else " ") for c in reversed(lines)])
+
+        return lines
+
+
+def collate_line(
+    line_chars: list,
+    tolerance=DEFAULT_X_TOLERANCE,
+) -> str:
+    coll = ""
+    last_x1 = None
+    for char in sorted(line_chars, key=itemgetter("x0")):
+        if (last_x1 is not None) and (char["x0"] > (last_x1 + tolerance)):
+            coll += " "
+        last_x1 = char["x1"]
+        coll += char["text"]
+    return coll
+
+
+def dedupe_chars(chars: list, tolerance=1) -> list:
+    """
+    Removes duplicate chars — those sharing the same text, fontname, size,
+    and positioning (within `tolerance`) as other characters in the set.
+    """
+    key = itemgetter("fontname", "size", "upright", "text")
+    pos_key = itemgetter("doctop", "x0")
+
+    def yield_unique_chars(chars: list):
+        sorted_chars = sorted(chars, key=key)
+        for grp, grp_chars in itertools.groupby(sorted_chars, key=key):
+            for y_cluster in cluster_objects(
+                list(grp_chars), itemgetter("doctop"), tolerance
+            ):
+                for x_cluster in cluster_objects(
+                    y_cluster, itemgetter("x0"), tolerance
+                ):
+                    yield sorted(x_cluster, key=pos_key)[0]
+
+    deduped = yield_unique_chars(chars)
+    return sorted(deduped, key=chars.index)
+
+
+def line_to_edge(line):
+    edge = dict(line)
+    edge["orientation"] = "h" if (line["top"] == line["bottom"]) else "v"
+    return edge
+
+
+def rect_to_edges(rect) -> list:
+    top, bottom, left, right = [dict(rect) for x in range(4)]
+    top.update(
+        {
+            "object_type": "rect_edge",
+            "height": 0,
+            "y0": rect["y1"],
+            "bottom": rect["top"],
+            "orientation": "h",
+        }
+    )
+    bottom.update(
+        {
+            "object_type": "rect_edge",
+            "height": 0,
+            "y1": rect["y0"],
+            "top": rect["top"] + rect["height"],
+            "doctop": rect["doctop"] + rect["height"],
+            "orientation": "h",
+        }
+    )
+    left.update(
+        {
+            "object_type": "rect_edge",
+            "width": 0,
+            "x1": rect["x0"],
+            "orientation": "v",
+        }
+    )
+    right.update(
+        {
+            "object_type": "rect_edge",
+            "width": 0,
+            "x0": rect["x1"],
+            "orientation": "v",
+        }
+    )
+    return [top, bottom, left, right]
+
+
+def curve_to_edges(curve) -> list:
+    point_pairs = zip(curve["pts"], curve["pts"][1:])
+    return [
+        {
+            "object_type": "curve_edge",
+            "x0": min(p0[0], p1[0]),
+            "x1": max(p0[0], p1[0]),
+            "top": min(p0[1], p1[1]),
+            "doctop": min(p0[1], p1[1]) + (curve["doctop"] - curve["top"]),
+            "bottom": max(p0[1], p1[1]),
+            "width": abs(p0[0] - p1[0]),
+            "height": abs(p0[1] - p1[1]),
+            "orientation": "v" if p0[0] == p1[0] else ("h" if p0[1] == p1[1] else None),
+        }
+        for p0, p1 in point_pairs
+    ]
+
+
+def obj_to_edges(obj) -> list:
+    t = obj["object_type"]
+    if "_edge" in t:
+        return [obj]
+    elif t == "line":
+        return [line_to_edge(obj)]
+    else:
+        return {"rect": rect_to_edges, "curve": curve_to_edges}[t](obj)
+
+
+def filter_edges(
+    edges,
+    orientation=None,
+    edge_type=None,
+    min_length=1,
+) -> list:
+    if orientation not in ("v", "h", None):
+        raise ValueError("Orientation must be 'v' or 'h'")
+
+    def test(e) -> bool:
+        dim = "height" if e["orientation"] == "v" else "width"
+        et_correct = e["object_type"] == edge_type if edge_type is not None else True
+        orient_correct = orientation is None or e["orientation"] == orientation
+        return bool(et_correct and orient_correct and (e[dim] >= min_length))
+
+    return list(filter(test, edges))
+
+
+def cluster_list(xs, tolerance=0) -> list:
+    if tolerance == 0:
+        return [[x] for x in sorted(xs)]
+    if len(xs) < 2:
+        return [[x] for x in sorted(xs)]
+    groups = []
+    xs = list(sorted(xs))
+    current_group = [xs[0]]
+    last = xs[0]
+    for x in xs[1:]:
+        if x <= (last + tolerance):
+            current_group.append(x)
+        else:
+            groups.append(current_group)
+            current_group = [x]
+        last = x
+    groups.append(current_group)
+    return groups
+
+
+def make_cluster_dict(values, tolerance) -> dict:
+    clusters = cluster_list(list(set(values)), tolerance)
+
+    nested_tuples = [
+        [(val, i) for val in value_cluster] for i, value_cluster in enumerate(clusters)
+    ]
+
+    return dict(itertools.chain(*nested_tuples))
+
+
+def cluster_objects(xs, key_fn, tolerance) -> list:
+    if not callable(key_fn):
+        key_fn = itemgetter(key_fn)
+
+    values = map(key_fn, xs)
+    cluster_dict = make_cluster_dict(values, tolerance)
+
+    get_0, get_1 = itemgetter(0), itemgetter(1)
+
+    cluster_tuples = sorted(((x, cluster_dict.get(key_fn(x))) for x in xs), key=get_1)
+
+    grouped = itertools.groupby(cluster_tuples, key=get_1)
+
+    return [list(map(get_0, v)) for k, v in grouped]
+
+
+def move_object(obj, axis: str, value):
+    assert axis in ("h", "v")
+    if axis == "h":
+        new_items = [
+            ("x0", obj["x0"] + value),
+            ("x1", obj["x1"] + value),
+        ]
+    if axis == "v":
+        new_items = [
+            ("top", obj["top"] + value),
+            ("bottom", obj["bottom"] + value),
+        ]
+        if "doctop" in obj:
+            new_items += [("doctop", obj["doctop"] + value)]
+        if "y0" in obj:
+            new_items += [
+                ("y0", obj["y0"] - value),
+                ("y1", obj["y1"] - value),
+            ]
+    return obj.__class__(tuple(obj.items()) + tuple(new_items))
+
+
+def snap_objects(objs, attr: str, tolerance) -> list:
+    axis = {"x0": "h", "x1": "h", "top": "v", "bottom": "v"}[attr]
+    list_objs = list(objs)
+    clusters = cluster_objects(list_objs, itemgetter(attr), tolerance)
+    avgs = [sum(map(itemgetter(attr), cluster)) / len(cluster) for cluster in clusters]
+    snapped_clusters = [
+        [move_object(obj, axis, avg - obj[attr]) for obj in cluster]
+        for cluster, avg in zip(clusters, avgs)
+    ]
+    return list(itertools.chain(*snapped_clusters))
+
+
+def snap_edges(
+    edges,
+    x_tolerance=DEFAULT_SNAP_TOLERANCE,
+    y_tolerance=DEFAULT_SNAP_TOLERANCE,
+):
+    """
+    Given a list of edges, snap any within `tolerance` pixels of one another
+    to their positional average.
+    """
+    by_orientation = {"v": [], "h": []}
+    for e in edges:
+        by_orientation[e["orientation"]].append(e)
+
+    snapped_v = snap_objects(by_orientation["v"], "x0", x_tolerance)
+    snapped_h = snap_objects(by_orientation["h"], "top", y_tolerance)
+    return snapped_v + snapped_h
+
+
+def resize_object(obj, key: str, value):
+    assert key in ("x0", "x1", "top", "bottom")
+    old_value = obj[key]
+    diff = value - old_value
+    new_items = [
+        (key, value),
+    ]
+    if key == "x0":
+        assert value <= obj["x1"]
+        new_items.append(("width", obj["x1"] - value))
+    elif key == "x1":
+        assert value >= obj["x0"]
+        new_items.append(("width", value - obj["x0"]))
+    elif key == "top":
+        assert value <= obj["bottom"]
+        new_items.append(("doctop", obj["doctop"] + diff))
+        new_items.append(("height", obj["height"] - diff))
+        if "y1" in obj:
+            new_items.append(("y1", obj["y1"] - diff))
+    elif key == "bottom":
+        assert value >= obj["top"]
+        new_items.append(("height", obj["height"] + diff))
+        if "y0" in obj:
+            new_items.append(("y0", obj["y0"] - diff))
+    return obj.__class__(tuple(obj.items()) + tuple(new_items))
+
+
+def join_edge_group(edges, orientation: str, tolerance=DEFAULT_JOIN_TOLERANCE):
+    """
+    Given a list of edges along the same infinite line, join those that
+    are within `tolerance` pixels of one another.
+    """
+    if orientation == "h":
+        min_prop, max_prop = "x0", "x1"
+    elif orientation == "v":
+        min_prop, max_prop = "top", "bottom"
+    else:
+        raise ValueError("Orientation must be 'v' or 'h'")
+
+    sorted_edges = list(sorted(edges, key=itemgetter(min_prop)))
+    joined = [sorted_edges[0]]
+    for e in sorted_edges[1:]:
+        last = joined[-1]
+        if e[min_prop] <= (last[max_prop] + tolerance):
+            if e[max_prop] > last[max_prop]:
+                # Extend current edge to new extremity
+                joined[-1] = resize_object(last, max_prop, e[max_prop])
+        else:
+            # Edge is separate from previous edges
+            joined.append(e)
+
+    return joined
+
+
+def merge_edges(
+    edges,
+    snap_x_tolerance,
+    snap_y_tolerance,
+    join_x_tolerance,
+    join_y_tolerance,
+):
+    """
+    Using the `snap_edges` and `join_edge_group` methods above,
+    merge a list of edges into a more "seamless" list.
+    """
+
+    def get_group(edge):
+        if edge["orientation"] == "h":
+            return ("h", edge["top"])
+        else:
+            return ("v", edge["x0"])
+
+    if snap_x_tolerance > 0 or snap_y_tolerance > 0:
+        edges = snap_edges(edges, snap_x_tolerance, snap_y_tolerance)
+
+    _sorted = sorted(edges, key=get_group)
+    edge_groups = itertools.groupby(_sorted, key=get_group)
+    edge_gen = (
+        join_edge_group(
+            items, k[0], (join_x_tolerance if k[0] == "h" else join_y_tolerance)
+        )
+        for k, items in edge_groups
+    )
+    edges = list(itertools.chain(*edge_gen))
+    return edges
+
+
+def bbox_to_rect(bbox) -> dict:
+    """
+    Return the rectangle (i.e a dict with keys "x0", "top", "x1",
+    "bottom") for an object.
+    """
+    return {"x0": bbox[0], "top": bbox[1], "x1": bbox[2], "bottom": bbox[3]}
+
+
+def objects_to_rect(objects) -> dict:
+    """
+    Given an iterable of objects, return the smallest rectangle (i.e. a
+    dict with "x0", "top", "x1", and "bottom" keys) that contains them
+    all.
+    """
+    return bbox_to_rect(objects_to_bbox(objects))
+
+
+def merge_bboxes(bboxes):
+    """
+    Given an iterable of bounding boxes, return the smallest bounding box
+    that contains them all.
+    """
+    x0, top, x1, bottom = zip(*bboxes)
+    return (min(x0), min(top), max(x1), max(bottom))
+
+
+def objects_to_bbox(objects):
+    """
+    Given an iterable of objects, return the smallest bounding box that
+    contains them all.
+    """
+    return merge_bboxes(map(bbox_getter, objects))
+
+
+def words_to_edges_h(words, word_threshold: int = DEFAULT_MIN_WORDS_HORIZONTAL):
+    """
+    Find (imaginary) horizontal lines that connect the tops
+    of at least `word_threshold` words.
+    """
+    by_top = cluster_objects(words, itemgetter("top"), 1)
+    large_clusters = filter(lambda x: len(x) >= word_threshold, by_top)
+    rects = list(map(objects_to_rect, large_clusters))
+    if len(rects) == 0:
+        return []
+    min_x0 = min(map(itemgetter("x0"), rects))
+    max_x1 = max(map(itemgetter("x1"), rects))
+
+    edges = []
+    for r in rects:
+        edges += [
+            # Top of text
+            {
+                "x0": min_x0,
+                "x1": max_x1,
+                "top": r["top"],
+                "bottom": r["top"],
+                "width": max_x1 - min_x0,
+                "orientation": "h",
+            },
+            # For each detected row, we also add the 'bottom' line.  This will
+            # generate extra edges, (some will be redundant with the next row
+            # 'top' line), but this catches the last row of every table.
+            {
+                "x0": min_x0,
+                "x1": max_x1,
+                "top": r["bottom"],
+                "bottom": r["bottom"],
+                "width": max_x1 - min_x0,
+                "orientation": "h",
+            },
+        ]
+
+    return edges
+
+
+def get_bbox_overlap(a, b):
+    a_left, a_top, a_right, a_bottom = a
+    b_left, b_top, b_right, b_bottom = b
+    o_left = max(a_left, b_left)
+    o_right = min(a_right, b_right)
+    o_bottom = min(a_bottom, b_bottom)
+    o_top = max(a_top, b_top)
+    o_width = o_right - o_left
+    o_height = o_bottom - o_top
+    if o_height >= 0 and o_width >= 0 and o_height + o_width > 0:
+        return (o_left, o_top, o_right, o_bottom)
+    else:
+        return None
+
+
+def words_to_edges_v(words, word_threshold: int = DEFAULT_MIN_WORDS_VERTICAL):
+    """
+    Find (imaginary) vertical lines that connect the left, right, or
+    center of at least `word_threshold` words.
+    """
+    # Find words that share the same left, right, or centerpoints
+    by_x0 = cluster_objects(words, itemgetter("x0"), 1)
+    by_x1 = cluster_objects(words, itemgetter("x1"), 1)
+
+    def get_center(word):
+        return float(word["x0"] + word["x1"]) / 2
+
+    by_center = cluster_objects(words, get_center, 1)
+    clusters = by_x0 + by_x1 + by_center
+
+    # Find the points that align with the most words
+    sorted_clusters = sorted(clusters, key=lambda x: -len(x))
+    large_clusters = filter(lambda x: len(x) >= word_threshold, sorted_clusters)
+
+    # For each of those points, find the bboxes fitting all matching words
+    bboxes = list(map(objects_to_bbox, large_clusters))
+
+    # Iterate through those bboxes, condensing overlapping bboxes
+    condensed_bboxes = []
+    for bbox in bboxes:
+        overlap = any(get_bbox_overlap(bbox, c) for c in condensed_bboxes)
+        if not overlap:
+            condensed_bboxes.append(bbox)
+
+    if not condensed_bboxes:
+        return []
+
+    condensed_rects = map(bbox_to_rect, condensed_bboxes)
+    sorted_rects = list(sorted(condensed_rects, key=itemgetter("x0")))
+
+    max_x1 = max(map(itemgetter("x1"), sorted_rects))
+    min_top = min(map(itemgetter("top"), sorted_rects))
+    max_bottom = max(map(itemgetter("bottom"), sorted_rects))
+
+    return [
+        {
+            "x0": b["x0"],
+            "x1": b["x0"],
+            "top": min_top,
+            "bottom": max_bottom,
+            "height": max_bottom - min_top,
+            "orientation": "v",
+        }
+        for b in sorted_rects
+    ] + [
+        {
+            "x0": max_x1,
+            "x1": max_x1,
+            "top": min_top,
+            "bottom": max_bottom,
+            "height": max_bottom - min_top,
+            "orientation": "v",
+        }
+    ]
+
+
+def edges_to_intersections(edges, x_tolerance=1, y_tolerance=1) -> dict:
+    """
+    Given a list of edges, return the points at which they intersect
+    within `tolerance` pixels.
+    """
+    intersections = {}
+    v_edges, h_edges = [
+        list(filter(lambda x: x["orientation"] == o, edges)) for o in ("v", "h")
+    ]
+    for v in sorted(v_edges, key=itemgetter("x0", "top")):
+        for h in sorted(h_edges, key=itemgetter("top", "x0")):
+            if (
+                (v["top"] <= (h["top"] + y_tolerance))
+                and (v["bottom"] >= (h["top"] - y_tolerance))
+                and (v["x0"] >= (h["x0"] - x_tolerance))
+                and (v["x0"] <= (h["x1"] + x_tolerance))
+            ):
+                vertex = (v["x0"], h["top"])
+                if vertex not in intersections:
+                    intersections[vertex] = {"v": [], "h": []}
+                intersections[vertex]["v"].append(v)
+                intersections[vertex]["h"].append(h)
+    return intersections
+
+
+def obj_to_bbox(obj):
+    """
+    Return the bounding box for an object.
+    """
+    return bbox_getter(obj)
+
+
+def intersections_to_cells(intersections):
+    """
+    Given a list of points (`intersections`), return all rectangular "cells"
+    that those points describe.
+
+    `intersections` should be a dictionary with (x0, top) tuples as keys,
+    and a list of edge objects as values. The edge objects should correspond
+    to the edges that touch the intersection.
+    """
+
+    def edge_connects(p1, p2) -> bool:
+        def edges_to_set(edges):
+            return set(map(obj_to_bbox, edges))
+
+        if p1[0] == p2[0]:
+            common = edges_to_set(intersections[p1]["v"]).intersection(
+                edges_to_set(intersections[p2]["v"])
+            )
+            if len(common):
+                return True
+
+        if p1[1] == p2[1]:
+            common = edges_to_set(intersections[p1]["h"]).intersection(
+                edges_to_set(intersections[p2]["h"])
+            )
+            if len(common):
+                return True
+        return False
+
+    points = list(sorted(intersections.keys()))
+    n_points = len(points)
+
+    def find_smallest_cell(points, i: int):
+        if i == n_points - 1:
+            return None
+        pt = points[i]
+        rest = points[i + 1 :]
+        # Get all the points directly below and directly right
+        below = [x for x in rest if x[0] == pt[0]]
+        right = [x for x in rest if x[1] == pt[1]]
+        for below_pt in below:
+            if not edge_connects(pt, below_pt):
+                continue
+
+            for right_pt in right:
+                if not edge_connects(pt, right_pt):
+                    continue
+
+                bottom_right = (right_pt[0], below_pt[1])
+
+                if (
+                    (bottom_right in intersections)
+                    and edge_connects(bottom_right, right_pt)
+                    and edge_connects(bottom_right, below_pt)
+                ):
+                    return (pt[0], pt[1], bottom_right[0], bottom_right[1])
+        return None
+
+    cell_gen = (find_smallest_cell(points, i) for i in range(len(points)))
+    return list(filter(None, cell_gen))
+
+
+def cells_to_tables(page, cells) -> list:
+    """
+    Given a list of bounding boxes (`cells`), return a list of tables that
+    hold those cells most simply (and contiguously).
+    """
+
+    def bbox_to_corners(bbox) -> tuple:
+        x0, top, x1, bottom = bbox
+        return ((x0, top), (x0, bottom), (x1, top), (x1, bottom))
+
+    remaining_cells = list(cells)
+
+    # Iterate through the cells found above, and assign them
+    # to contiguous tables
+
+    current_corners = set()
+    current_cells = []
+
+    tables = []
+    while len(remaining_cells):
+        initial_cell_count = len(current_cells)
+        for cell in list(remaining_cells):
+            cell_corners = bbox_to_corners(cell)
+            # If we're just starting a table ...
+            if len(current_cells) == 0:
+                # ... immediately assign it to the empty group
+                current_corners |= set(cell_corners)
+                current_cells.append(cell)
+                remaining_cells.remove(cell)
+            else:
+                # How many corners does this table share with the current group?
+                corner_count = sum(c in current_corners for c in cell_corners)
+
+                # If touching on at least one corner...
+                if corner_count > 0:
+                    # ... assign it to the current group
+                    current_corners |= set(cell_corners)
+                    current_cells.append(cell)
+                    remaining_cells.remove(cell)
+
+        # If this iteration did not find any more cells to append...
+        if len(current_cells) == initial_cell_count:
+            # ... start a new cell group
+            tables.append(list(current_cells))
+            current_corners.clear()
+            current_cells.clear()
+
+    # Once we have exhausting the list of cells ...
+
+    # ... and we have a cell group that has not been stored
+    if len(current_cells):
+        # ... store it.
+        tables.append(list(current_cells))
+
+    # PyMuPDF modification:
+    # Remove tables without text or having only 1 column
+    for i in range(len(tables) - 1, -1, -1):
+        r = EMPTY_RECT()
+        x1_vals = set()
+        x0_vals = set()
+        for c in tables[i]:
+            r |= c
+            x1_vals.add(c[2])
+            x0_vals.add(c[0])
+        if (
+            len(x1_vals) < 2
+            or len(x0_vals) < 2
+            or white_spaces.issuperset(
+                page.get_textbox(
+                    r,
+                    textpage=TEXTPAGE,
+                )
+            )
+        ):
+            del tables[i]
+
+    # Sort the tables top-to-bottom-left-to-right based on the value of the
+    # topmost-and-then-leftmost coordinate of a table.
+    _sorted = sorted(tables, key=lambda t: min((c[1], c[0]) for c in t))
+    return _sorted
+
+
+class CellGroup:
+    def __init__(self, cells):
+        self.cells = cells
+        self.bbox = (
+            min(map(itemgetter(0), filter(None, cells))),
+            min(map(itemgetter(1), filter(None, cells))),
+            max(map(itemgetter(2), filter(None, cells))),
+            max(map(itemgetter(3), filter(None, cells))),
+        )
+
+
+class TableRow(CellGroup):
+    pass
+
+
+class TableHeader:
+    """PyMuPDF extension containing the identified table header."""
+
+    def __init__(self, bbox, cells, names, above):
+        self.bbox = bbox
+        self.cells = cells
+        self.names = names
+        self.external = above
+
+
+class Table:
+    def __init__(self, page, cells):
+        self.page = page
+        self.cells = cells
+        self.header = self._get_header()  # PyMuPDF extension
+
+    @property
+    def bbox(self):
+        c = self.cells
+        return (
+            min(map(itemgetter(0), c)),
+            min(map(itemgetter(1), c)),
+            max(map(itemgetter(2), c)),
+            max(map(itemgetter(3), c)),
+        )
+
+    @property
+    def rows(self) -> list:
+        _sorted = sorted(self.cells, key=itemgetter(1, 0))
+        xs = list(sorted(set(map(itemgetter(0), self.cells))))
+        rows = []
+        for y, row_cells in itertools.groupby(_sorted, itemgetter(1)):
+            xdict = {cell[0]: cell for cell in row_cells}
+            row = TableRow([xdict.get(x) for x in xs])
+            rows.append(row)
+        return rows
+
+    @property
+    def row_count(self) -> int:  # PyMuPDF extension
+        return len(self.rows)
+
+    @property
+    def col_count(self) -> int:  # PyMuPDF extension
+        return max([len(r.cells) for r in self.rows])
+
+    def extract(self, **kwargs) -> list:
+        chars = CHARS
+        table_arr = []
+
+        def char_in_bbox(char, bbox) -> bool:
+            v_mid = (char["top"] + char["bottom"]) / 2
+            h_mid = (char["x0"] + char["x1"]) / 2
+            x0, top, x1, bottom = bbox
+            return bool(
+                (h_mid >= x0) and (h_mid < x1) and (v_mid >= top) and (v_mid < bottom)
+            )
+
+        for row in self.rows:
+            arr = []
+            row_chars = [char for char in chars if char_in_bbox(char, row.bbox)]
+
+            for cell in row.cells:
+                if cell is None:
+                    cell_text = None
+                else:
+                    cell_chars = [
+                        char for char in row_chars if char_in_bbox(char, cell)
+                    ]
+
+                    if len(cell_chars):
+                        kwargs["x_shift"] = cell[0]
+                        kwargs["y_shift"] = cell[1]
+                        if "layout" in kwargs:
+                            kwargs["layout_width"] = cell[2] - cell[0]
+                            kwargs["layout_height"] = cell[3] - cell[1]
+                        cell_text = extract_text(cell_chars, **kwargs)
+                    else:
+                        cell_text = ""
+                arr.append(cell_text)
+            table_arr.append(arr)
+
+        return table_arr
+
+    def to_markdown(self, clean=False, fill_empty=True):
+        """Output table content as a string in Github-markdown format.
+
+        If "clean" then markdown syntax is removed from cell content.
+        If "fill_empty" then cell content None is replaced by the values
+        above (columns) or left (rows) in an effort to approximate row and
+        columns spans.
+
+        """
+        output = "|"
+        rows = self.row_count
+        cols = self.col_count
+        cells = self.extract()[:]  # make local copy of table text content
+
+        if fill_empty:  # fill "None" cells where possible
+
+            # for rows, copy content from left to right
+            for j in range(rows):
+                for i in range(cols - 1):
+                    if cells[j][i + 1] is None:
+                        cells[j][i + 1] = cells[j][i]
+
+            # for columns, copy top to bottom
+            for i in range(cols):
+                for j in range(rows - 1):
+                    if cells[j + 1][i] is None:
+                        cells[j + 1][i] = cells[j][i]
+
+        # generate header string and MD separator
+        for i, name in enumerate(self.header.names):
+            if not name:  # generate a name if empty
+                name = f"Col{i+1}"
+            name = name.replace("\n", "
    ")  # use HTML line breaks
+            if clean:  # remove sensitive syntax
+                name = html.escape(name.replace("-", "-"))
+            output += name + "|"
+
+        output += "\n"
+        # insert GitHub header line separator
+        output += "|" + "|".join("---" for i in range(self.col_count)) + "|\n"
+
+        # skip first row in details if header is part of the table
+        j = 0 if self.header.external else 1
+
+        # iterate over detail rows
+        for row in cells[j:]:
+            line = "|"
+            for i, cell in enumerate(row):
+                # replace None cells with empty string
+                # use HTML line break tag
+                cell = "" if not cell else cell.replace("\n", "
    ")
+                if clean:  # remove sensitive syntax
+                    cell = html.escape(cell.replace("-", "-"))
+                line += cell + "|"
+            line += "\n"
+            output += line
+        return output + "\n"
+
+    def to_pandas(self, **kwargs):
+        """Return a pandas DataFrame version of the table."""
+        try:
+            import pandas as pd
+        except ModuleNotFoundError:
+            message("Package 'pandas' is not installed")
+            raise
+
+        pd_dict = {}
+        extract = self.extract()
+        hdr = self.header
+        names = self.header.names
+        hdr_len = len(names)
+        # ensure uniqueness of column names
+        for i in range(hdr_len):
+            name = names[i]
+            if not name:
+                names[i] = f"Col{i}"
+        if hdr_len != len(set(names)):
+            for i in range(hdr_len):
+                name = names[i]
+                if name != f"Col{i}":
+                    names[i] = f"{i}-{name}"
+
+        if not hdr.external:  # header is part of 'extract'
+            extract = extract[1:]
+
+        for i in range(hdr_len):
+            key = names[i]
+            value = []
+            for j in range(len(extract)):
+                value.append(extract[j][i])
+            pd_dict[key] = value
+
+        return pd.DataFrame(pd_dict)
+
+    def _get_header(self, y_tolerance=3):
+        """Identify the table header.
+
+        *** PyMuPDF extension. ***
+
+        Starting from the first line above the table upwards, check if it
+        qualifies to be part of the table header.
+
+        Criteria include:
+        * A one-line table never has an extra header.
+        * Column borders must not intersect any word. If this happens, all
+          text of this line and above of it is ignored.
+        * No excess inter-line distance: If a line further up has a distance
+          of more than 1.5 times of its font size, it will be ignored and
+          all lines above of it.
+        * Must have same text properties.
+        * Starting with the top table line, a bold text property cannot change
+          back to non-bold.
+
+        If not all criteria are met (or there is no text above the table),
+        the first table row is assumed to be the header.
+        """
+        page = self.page
+        y_delta = y_tolerance
+
+        def top_row_bg_color(self):
+            """
+            Compare top row background color with color of same-sized bbox
+            above. If different, return True indicating that the original
+            table top row is already the header.
+            """
+            bbox0 = Rect(self.rows[0].bbox)
+            bboxt = bbox0 + (0, -bbox0.height, 0, -bbox0.height)  # area above
+            top_color0 = page.get_pixmap(clip=bbox0).color_topusage()[1]
+            top_colort = page.get_pixmap(clip=bboxt).color_topusage()[1]
+            if top_color0 != top_colort:
+                return True  # top row is header
+            return False
+
+        def row_has_bold(bbox):
+            """Check if a row contains some bold text.
+
+            If e.g. true for the top row, then it will be used as (internal)
+            column header row if any of the following is true:
+            * the previous (above) text line has no bold span
+            * the second table row text has no bold span
+
+            Returns True if any spans are bold else False.
+            """
+            blocks = page.get_text("dict", flags=TEXTFLAGS_TEXT, clip=bbox)["blocks"]
+            spans = [s for b in blocks for l in b["lines"] for s in l["spans"]]
+
+            return any(s["flags"] & TEXT_FONT_BOLD for s in spans)
+
+        try:
+            row = self.rows[0]
+            cells = row.cells
+            bbox = Rect(row.bbox)
+        except IndexError:  # this table has no rows
+            return None
+
+        # return this if we determine that the top row is the header
+        header_top_row = TableHeader(bbox, cells, self.extract()[0], False)
+
+        # 1-line tables have no extra header
+        if len(self.rows) < 2:
+            return header_top_row
+
+        # 1-column tables have no extra header
+        if len(cells) < 2:
+            return header_top_row
+
+        # assume top row is the header if second row is empty
+        row2 = self.rows[1]  # second row
+        if all(c is None for c in row2.cells):  # no valid cell bboxes in row2
+            return header_top_row
+
+        # Special check: is top row bold?
+        top_row_bold = row_has_bold(bbox)
+
+        # assume top row is header if it is bold and any cell
+        # of 2nd row is non-bold
+        if top_row_bold and not row_has_bold(row2.bbox):
+            return header_top_row
+
+        if top_row_bg_color(self):
+            # if area above top row has a different background color,
+            # then top row is already the header
+            return header_top_row
+
+        # column coordinates (x1 values) in top row
+        col_x = [c[2] if c is not None else None for c in cells[:-1]]
+
+        # clip = page area above the table
+        # We will inspect this area for text qualifying as column header.
+        clip = +bbox  # take row 0 bbox
+        clip.y0 = 0  # start at top of page
+        clip.y1 = bbox.y0  # end at top of table
+
+        blocks = page.get_text("dict", clip=clip, flags=TEXTFLAGS_TEXT)["blocks"]
+        # non-empty, non-superscript spans above table, sorted descending by y1
+        spans = sorted(
+            [
+                s
+                for b in blocks
+                for l in b["lines"]
+                for s in l["spans"]
+                if not (
+                    white_spaces.issuperset(s["text"])
+                    or s["flags"] & TEXT_FONT_SUPERSCRIPT
+                )
+            ],
+            key=lambda s: s["bbox"][3],
+            reverse=True,
+        )
+
+        select = []  # y1 coordinates above, sorted descending
+        line_heights = []  # line heights above, sorted descending
+        line_bolds = []  # bold indicator per line above, same sorting
+
+        # walk through the spans and fill above 3 lists
+        for i in range(len(spans)):
+            s = spans[i]
+            y1 = s["bbox"][3]  # span bottom
+            h = y1 - s["bbox"][1]  # span bbox height
+            bold = s["flags"] & TEXT_FONT_BOLD
+
+            # use first item to start the lists
+            if i == 0:
+                select.append(y1)
+                line_heights.append(h)
+                line_bolds.append(bold)
+                continue
+
+            # get previous items from the 3 lists
+            y0 = select[-1]
+            h0 = line_heights[-1]
+            bold0 = line_bolds[-1]
+
+            if bold0 and not bold:
+                break  # stop if switching from bold to non-bold
+
+            # if fitting in height of previous span, modify bbox
+            if y0 - y1 <= y_delta or abs((y0 - h0) - s["bbox"][1]) <= y_delta:
+                s["bbox"] = (s["bbox"][0], y0 - h0, s["bbox"][2], y0)
+                spans[i] = s
+                if bold:
+                    line_bolds[-1] = bold
+                continue
+            elif y0 - y1 > 1.5 * h0:
+                break  # stop if distance to previous line too large
+            select.append(y1)
+            line_heights.append(h)
+            line_bolds.append(bold)
+
+        if select == []:  # nothing above the table?
+            return header_top_row
+
+        select = select[:5]  # accept up to 5 lines for an external header
+
+        # assume top row as header if text above is too far away
+        if bbox.y0 - select[0] >= line_heights[0]:
+            return header_top_row
+
+        # accept top row as header if bold, but line above is not
+        if top_row_bold and not line_bolds[0]:
+            return header_top_row
+
+        if spans == []:  # nothing left above the table, return top row
+            return header_top_row
+
+        # re-compute clip above table
+        nclip = EMPTY_RECT()
+        for s in [s for s in spans if s["bbox"][3] >= select[-1]]:
+            nclip |= s["bbox"]
+        if not nclip.is_empty:
+            clip = nclip
+
+        clip.y1 = bbox.y0  # make sure we still include every word above
+
+        # Confirm that no word in clip is intersecting a column separator
+        word_rects = [Rect(w[:4]) for w in page.get_text("words", clip=clip)]
+        word_tops = sorted(list(set([r[1] for r in word_rects])), reverse=True)
+
+        select = []
+
+        # exclude lines with words that intersect a column border
+        for top in word_tops:
+            intersecting = [
+                (x, r)
+                for x in col_x
+                if x is not None
+                for r in word_rects
+                if r[1] == top and r[0] < x and r[2] > x
+            ]
+            if intersecting == []:
+                select.append(top)
+            else:  # detected a word crossing a column border
+                break
+
+        if select == []:  # nothing left over: return first row
+            return header_top_row
+
+        hdr_bbox = +clip  # compute the header cells
+        hdr_bbox.y0 = select[-1]  # hdr_bbox top is smallest top coord of words
+        hdr_cells = [
+            (c[0], hdr_bbox.y0, c[2], hdr_bbox.y1) if c is not None else None
+            for c in cells
+        ]
+
+        # adjust left/right of header bbox
+        hdr_bbox.x0 = self.bbox[0]
+        hdr_bbox.x1 = self.bbox[2]
+
+        # column names: no line breaks, no excess spaces
+        hdr_names = [
+            (
+                page.get_textbox(c).replace("\n", " ").replace("  ", " ").strip()
+                if c is not None
+                else ""
+            )
+            for c in hdr_cells
+        ]
+        return TableHeader(tuple(hdr_bbox), hdr_cells, hdr_names, True)
+
+
+@dataclass
+class TableSettings:
+    vertical_strategy: str = "lines"
+    horizontal_strategy: str = "lines"
+    explicit_vertical_lines: list = None
+    explicit_horizontal_lines: list = None
+    snap_tolerance: float = DEFAULT_SNAP_TOLERANCE
+    snap_x_tolerance: float = UNSET
+    snap_y_tolerance: float = UNSET
+    join_tolerance: float = DEFAULT_JOIN_TOLERANCE
+    join_x_tolerance: float = UNSET
+    join_y_tolerance: float = UNSET
+    edge_min_length: float = 3
+    min_words_vertical: float = DEFAULT_MIN_WORDS_VERTICAL
+    min_words_horizontal: float = DEFAULT_MIN_WORDS_HORIZONTAL
+    intersection_tolerance: float = 3
+    intersection_x_tolerance: float = UNSET
+    intersection_y_tolerance: float = UNSET
+    text_settings: dict = None
+
+    def __post_init__(self) -> "TableSettings":
+        """Clean up user-provided table settings.
+
+        Validates that the table settings provided consists of acceptable values and
+        returns a cleaned up version. The cleaned up version fills out the missing
+        values with the default values in the provided settings.
+
+        TODO: Can be further used to validate that the values are of the correct
+            type. For example, raising a value error when a non-boolean input is
+            provided for the key ``keep_blank_chars``.
+
+        :param table_settings: User-provided table settings.
+        :returns: A cleaned up version of the user-provided table settings.
+        :raises ValueError: When an unrecognised key is provided.
+        """
+
+        for setting in NON_NEGATIVE_SETTINGS:
+            if (getattr(self, setting) or 0) < 0:
+                raise ValueError(f"Table setting '{setting}' cannot be negative")
+
+        for orientation in ["horizontal", "vertical"]:
+            strategy = getattr(self, orientation + "_strategy")
+            if strategy not in TABLE_STRATEGIES:
+                raise ValueError(
+                    f"{orientation}_strategy must be one of"
+                    f'{{{",".join(TABLE_STRATEGIES)}}}'
+                )
+
+        if self.text_settings is None:
+            self.text_settings = {}
+
+        # This next section is for backwards compatibility
+        for attr in ["x_tolerance", "y_tolerance"]:
+            if attr not in self.text_settings:
+                self.text_settings[attr] = self.text_settings.get("tolerance", 3)
+
+        if "tolerance" in self.text_settings:
+            del self.text_settings["tolerance"]
+        # End of that section
+
+        for attr, fallback in [
+            ("snap_x_tolerance", "snap_tolerance"),
+            ("snap_y_tolerance", "snap_tolerance"),
+            ("join_x_tolerance", "join_tolerance"),
+            ("join_y_tolerance", "join_tolerance"),
+            ("intersection_x_tolerance", "intersection_tolerance"),
+            ("intersection_y_tolerance", "intersection_tolerance"),
+        ]:
+            if getattr(self, attr) is UNSET:
+                setattr(self, attr, getattr(self, fallback))
+
+        return self
+
+    @classmethod
+    def resolve(cls, settings=None):
+        if settings is None:
+            return cls()
+        elif isinstance(settings, cls):
+            return settings
+        elif isinstance(settings, dict):
+            core_settings = {}
+            text_settings = {}
+            for k, v in settings.items():
+                if k[:5] == "text_":
+                    text_settings[k[5:]] = v
+                else:
+                    core_settings[k] = v
+            core_settings["text_settings"] = text_settings
+            return cls(**core_settings)
+        else:
+            raise ValueError(f"Cannot resolve settings: {settings}")
+
+
+class TableFinder:
+    """
+    Given a PDF page, find plausible table structures.
+
+    Largely borrowed from Anssi Nurminen's master's thesis:
+    http://dspace.cc.tut.fi/dpub/bitstream/handle/123456789/21520/Nurminen.pdf?sequence=3
+
+    ... and inspired by Tabula:
+    https://github.com/tabulapdf/tabula-extractor/issues/16
+    """
+
+    def __init__(self, page, settings=None):
+        self.page = weakref.proxy(page)
+        self.settings = TableSettings.resolve(settings)
+        self.edges = self.get_edges()
+        self.intersections = edges_to_intersections(
+            self.edges,
+            self.settings.intersection_x_tolerance,
+            self.settings.intersection_y_tolerance,
+        )
+        self.cells = intersections_to_cells(self.intersections)
+        self.tables = [
+            Table(self.page, cell_group)
+            for cell_group in cells_to_tables(self.page, self.cells)
+        ]
+
+    def get_edges(self) -> list:
+        settings = self.settings
+
+        for orientation in ["vertical", "horizontal"]:
+            strategy = getattr(settings, orientation + "_strategy")
+            if strategy == "explicit":
+                lines = getattr(settings, "explicit_" + orientation + "_lines")
+                if len(lines) < 2:
+                    raise ValueError(
+                        f"If {orientation}_strategy == 'explicit', "
+                        f"explicit_{orientation}_lines "
+                        f"must be specified as a list/tuple of two or more "
+                        f"floats/ints."
+                    )
+
+        v_strat = settings.vertical_strategy
+        h_strat = settings.horizontal_strategy
+
+        if v_strat == "text" or h_strat == "text":
+            words = extract_words(CHARS, **(settings.text_settings or {}))
+        else:
+            words = []
+
+        v_explicit = []
+        for desc in settings.explicit_vertical_lines or []:
+            if isinstance(desc, dict):
+                for e in obj_to_edges(desc):
+                    if e["orientation"] == "v":
+                        v_explicit.append(e)
+            else:
+                v_explicit.append(
+                    {
+                        "x0": desc,
+                        "x1": desc,
+                        "top": self.page.rect[1],
+                        "bottom": self.page.rect[3],
+                        "height": self.page.rect[3] - self.page.rect[1],
+                        "orientation": "v",
+                    }
+                )
+
+        if v_strat == "lines":
+            v_base = filter_edges(EDGES, "v")
+        elif v_strat == "lines_strict":
+            v_base = filter_edges(EDGES, "v", edge_type="line")
+        elif v_strat == "text":
+            v_base = words_to_edges_v(words, word_threshold=settings.min_words_vertical)
+        elif v_strat == "explicit":
+            v_base = []
+        else:
+            v_base = []
+
+        v = v_base + v_explicit
+
+        h_explicit = []
+        for desc in settings.explicit_horizontal_lines or []:
+            if isinstance(desc, dict):
+                for e in obj_to_edges(desc):
+                    if e["orientation"] == "h":
+                        h_explicit.append(e)
+            else:
+                h_explicit.append(
+                    {
+                        "x0": self.page.rect[0],
+                        "x1": self.page.rect[2],
+                        "width": self.page.rect[2] - self.page.rect[0],
+                        "top": desc,
+                        "bottom": desc,
+                        "orientation": "h",
+                    }
+                )
+
+        if h_strat == "lines":
+            h_base = filter_edges(EDGES, "h")
+        elif h_strat == "lines_strict":
+            h_base = filter_edges(EDGES, "h", edge_type="line")
+        elif h_strat == "text":
+            h_base = words_to_edges_h(
+                words, word_threshold=settings.min_words_horizontal
+            )
+        elif h_strat == "explicit":
+            h_base = []
+        else:
+            h_base = []
+
+        h = h_base + h_explicit
+
+        edges = list(v) + list(h)
+
+        edges = merge_edges(
+            edges,
+            snap_x_tolerance=settings.snap_x_tolerance,
+            snap_y_tolerance=settings.snap_y_tolerance,
+            join_x_tolerance=settings.join_x_tolerance,
+            join_y_tolerance=settings.join_y_tolerance,
+        )
+
+        return filter_edges(edges, min_length=settings.edge_min_length)
+
+    def __getitem__(self, i):
+        tcount = len(self.tables)
+        if i >= tcount:
+            raise IndexError("table not on page")
+        while i < 0:
+            i += tcount
+        return self.tables[i]
+
+
+"""
+Start of PyMuPDF interface code.
+The following functions are executed when "page.find_tables()" is called.
+
+* make_chars: Fills the CHARS list with text character information extracted
+              via "rawdict" text extraction. Items in CHARS are formatted
+              as expected by the table code.
+* make_edges: Fills the EDGES list with vector graphic information extracted
+              via "get_drawings". Items in EDGES are formatted as expected
+              by the table code.
+
+The lists CHARS and EDGES are used to replace respective document access
+of pdfplumber or, respectively pdfminer.
+The table code has been modified to use these lists instead of accessing
+page information themselves.
+"""
+
+
+# -----------------------------------------------------------------------------
+# Extract all page characters to fill the CHARS list
+# -----------------------------------------------------------------------------
+def make_chars(page, clip=None):
+    """Extract text as "rawdict" to fill CHARS."""
+    global TEXTPAGE
+    page_number = page.number + 1
+    page_height = page.rect.height
+    ctm = page.transformation_matrix
+    TEXTPAGE = page.get_textpage(clip=clip, flags=TEXTFLAGS_TEXT)
+    blocks = page.get_text("rawdict", textpage=TEXTPAGE)["blocks"]
+    doctop_base = page_height * page.number
+    for block in blocks:
+        for line in block["lines"]:
+            ldir = line["dir"]  # = (cosine, sine) of angle
+            ldir = (round(ldir[0], 4), round(ldir[1], 4))
+            matrix = Matrix(ldir[0], -ldir[1], ldir[1], ldir[0], 0, 0)
+            if ldir[1] == 0:
+                upright = True
+            else:
+                upright = False
+            for span in sorted(line["spans"], key=lambda s: s["bbox"][0]):
+                fontname = span["font"]
+                fontsize = span["size"]
+                color = sRGB_to_pdf(span["color"])
+                for char in sorted(span["chars"], key=lambda c: c["bbox"][0]):
+                    bbox = Rect(char["bbox"])
+                    bbox_ctm = bbox * ctm
+                    origin = Point(char["origin"]) * ctm
+                    matrix.e = origin.x
+                    matrix.f = origin.y
+                    text = char["c"]
+                    char_dict = {
+                        "adv": bbox.x1 - bbox.x0 if upright else bbox.y1 - bbox.y0,
+                        "bottom": bbox.y1,
+                        "doctop": bbox.y0 + doctop_base,
+                        "fontname": fontname,
+                        "height": bbox.y1 - bbox.y0,
+                        "matrix": tuple(matrix),
+                        "ncs": "DeviceRGB",
+                        "non_stroking_color": color,
+                        "non_stroking_pattern": None,
+                        "object_type": "char",
+                        "page_number": page_number,
+                        "size": fontsize if upright else bbox.y1 - bbox.y0,
+                        "stroking_color": color,
+                        "stroking_pattern": None,
+                        "text": text,
+                        "top": bbox.y0,
+                        "upright": upright,
+                        "width": bbox.x1 - bbox.x0,
+                        "x0": bbox.x0,
+                        "x1": bbox.x1,
+                        "y0": bbox_ctm.y0,
+                        "y1": bbox_ctm.y1,
+                    }
+                    CHARS.append(char_dict)
+
+
+# ------------------------------------------------------------------------
+# Extract all page vector graphics to fill the EDGES list.
+# We are ignoring Bézier curves completely and are converting everything
+# else to lines.
+# ------------------------------------------------------------------------
+def make_edges(page, clip=None, tset=None, paths=None, add_lines=None, add_boxes=None):
+    snap_x = tset.snap_x_tolerance
+    snap_y = tset.snap_y_tolerance
+    min_length = tset.edge_min_length
+    lines_strict = (
+        tset.vertical_strategy == "lines_strict"
+        or tset.horizontal_strategy == "lines_strict"
+    )
+    page_height = page.rect.height
+    doctop_basis = page.number * page_height
+    page_number = page.number + 1
+    prect = page.rect
+    if page.rotation in (90, 270):
+        w, h = prect.br
+        prect = Rect(0, 0, h, w)
+    if clip is not None:
+        clip = Rect(clip)
+    else:
+        clip = prect
+
+    def are_neighbors(r1, r2):
+        """Detect whether r1, r2 are neighbors.
+
+        Defined as:
+        The minimum distance between points of r1 and points of r2 is not
+        larger than some delta.
+
+        This check supports empty rect-likes and thus also lines.
+
+        Note:
+        This type of check is MUCH faster than native Rect containment checks.
+        """
+        if (  # check if x-coordinates of r1 are within those of r2
+            r2.x0 - snap_x <= r1.x0 <= r2.x1 + snap_x
+            or r2.x0 - snap_x <= r1.x1 <= r2.x1 + snap_x
+        ) and (  # ... same for y-coordinates
+            r2.y0 - snap_y <= r1.y0 <= r2.y1 + snap_y
+            or r2.y0 - snap_y <= r1.y1 <= r2.y1 + snap_y
+        ):
+            return True
+
+        # same check with r1 / r2 exchanging their roles (this is necessary!)
+        if (
+            r1.x0 - snap_x <= r2.x0 <= r1.x1 + snap_x
+            or r1.x0 - snap_x <= r2.x1 <= r1.x1 + snap_x
+        ) and (
+            r1.y0 - snap_y <= r2.y0 <= r1.y1 + snap_y
+            or r1.y0 - snap_y <= r2.y1 <= r1.y1 + snap_y
+        ):
+            return True
+        return False
+
+    def clean_graphics(npaths=None):
+        """Detect and join rectangles of "connected" vector graphics."""
+        if npaths is None:
+            allpaths = page.get_drawings()
+        else:  # accept passed-in vector graphics
+            allpaths = npaths[:]  # paths relevant for table detection
+        paths = []
+        for p in allpaths:
+            # If only looking at lines, we ignore fill-only paths,
+            # except simulated lines (i.e. small width or height).
+            if (
+                lines_strict
+                and p["type"] == "f"
+                and p["rect"].width > snap_x
+                and p["rect"].height > snap_y
+            ):
+                continue
+            paths.append(p)
+
+        # start with all vector graphics rectangles
+        prects = sorted(set([p["rect"] for p in paths]), key=lambda r: (r.y1, r.x0))
+        new_rects = []  # the final list of joined rectangles
+        # ----------------------------------------------------------------
+        # Strategy: Join rectangles that "almost touch" each other.
+        # Extend first rectangle with any other that is a "neighbor".
+        # Then move it to the final list and continue with the rest.
+        # ----------------------------------------------------------------
+        while prects:  # the algorithm will empty this list
+            prect0 = prects[0]  # copy of first rectangle (performance reasons!)
+            repeat = True
+            while repeat:  # this loop extends first rect in list
+                repeat = False  # set to true again if some other rect touches
+                for i in range(len(prects) - 1, 0, -1):  # run backwards
+                    if are_neighbors(prect0, prects[i]):  # close enough to rect 0?
+                        prect0 |= prects[i].tl  # extend rect 0
+                        prect0 |= prects[i].br  # extend rect 0
+                        del prects[i]  # delete this rect
+                        repeat = True  # keep checking the rest
+
+            # move rect 0 over to result list if there is some text in it
+            if not white_spaces.issuperset(page.get_textbox(prect0, textpage=TEXTPAGE)):
+                # contains text, so accept it as a table bbox candidate
+                new_rects.append(prect0)
+            del prects[0]  # remove from rect list
+
+        return new_rects, paths
+
+    bboxes, paths = clean_graphics(npaths=paths)
+
+    def is_parallel(p1, p2):
+        """Check if line is roughly axis-parallel."""
+        if abs(p1.x - p2.x) <= snap_x or abs(p1.y - p2.y) <= snap_y:
+            return True
+        return False
+
+    def make_line(p, p1, p2, clip):
+        """Given 2 points, make a line dictionary for table detection."""
+        if not is_parallel(p1, p2):  # only accepting axis-parallel lines
+            return {}
+        # compute the extremal values
+        x0 = min(p1.x, p2.x)
+        x1 = max(p1.x, p2.x)
+        y0 = min(p1.y, p2.y)
+        y1 = max(p1.y, p2.y)
+
+        # check for outside clip
+        if x0 > clip.x1 or x1 < clip.x0 or y0 > clip.y1 or y1 < clip.y0:
+            return {}
+
+        if x0 < clip.x0:
+            x0 = clip.x0  # adjust to clip boundary
+
+        if x1 > clip.x1:
+            x1 = clip.x1  # adjust to clip boundary
+
+        if y0 < clip.y0:
+            y0 = clip.y0  # adjust to clip boundary
+
+        if y1 > clip.y1:
+            y1 = clip.y1  # adjust to clip boundary
+
+        width = x1 - x0  # from adjusted values
+        height = y1 - y0  # from adjusted values
+        if width == height == 0:
+            return {}  # nothing left to deal with
+        line_dict = {
+            "x0": x0,
+            "y0": page_height - y0,
+            "x1": x1,
+            "y1": page_height - y1,
+            "width": width,
+            "height": height,
+            "pts": [(x0, y0), (x1, y1)],
+            "linewidth": p["width"],
+            "stroke": True,
+            "fill": False,
+            "evenodd": False,
+            "stroking_color": p["color"] if p["color"] else p["fill"],
+            "non_stroking_color": None,
+            "object_type": "line",
+            "page_number": page_number,
+            "stroking_pattern": None,
+            "non_stroking_pattern": None,
+            "top": y0,
+            "bottom": y1,
+            "doctop": y0 + doctop_basis,
+        }
+        return line_dict
+
+    for p in paths:
+        items = p["items"]  # items in this path
+
+        # if 'closePath', add a line from last to first point
+        if p["closePath"] and items[0][0] == "l" and items[-1][0] == "l":
+            items.append(("l", items[-1][2], items[0][1]))
+
+        for i in items:
+            if i[0] not in ("l", "re", "qu"):
+                continue  # ignore anything else
+
+            if i[0] == "l":  # a line
+                p1, p2 = i[1:]
+                line_dict = make_line(p, p1, p2, clip)
+                if line_dict:
+                    EDGES.append(line_to_edge(line_dict))
+
+            elif i[0] == "re":
+                # A rectangle: decompose into 4 lines, but filter out
+                # the ones that simulate a line
+                rect = i[1].normalize()  # normalize the rectangle
+
+                if (
+                    rect.width <= min_length and rect.width < rect.height
+                ):  # simulates a vertical line
+                    x = abs(rect.x1 + rect.x0) / 2  # take middle value for x
+                    p1 = Point(x, rect.y0)
+                    p2 = Point(x, rect.y1)
+                    line_dict = make_line(p, p1, p2, clip)
+                    if line_dict:
+                        EDGES.append(line_to_edge(line_dict))
+                    continue
+
+                if (
+                    rect.height <= min_length and rect.height < rect.width
+                ):  # simulates a horizontal line
+                    y = abs(rect.y1 + rect.y0) / 2  # take middle value for y
+                    p1 = Point(rect.x0, y)
+                    p2 = Point(rect.x1, y)
+                    line_dict = make_line(p, p1, p2, clip)
+                    if line_dict:
+                        EDGES.append(line_to_edge(line_dict))
+                    continue
+
+                line_dict = make_line(p, rect.tl, rect.bl, clip)
+                if line_dict:
+                    EDGES.append(line_to_edge(line_dict))
+
+                line_dict = make_line(p, rect.bl, rect.br, clip)
+                if line_dict:
+                    EDGES.append(line_to_edge(line_dict))
+
+                line_dict = make_line(p, rect.br, rect.tr, clip)
+                if line_dict:
+                    EDGES.append(line_to_edge(line_dict))
+
+                line_dict = make_line(p, rect.tr, rect.tl, clip)
+                if line_dict:
+                    EDGES.append(line_to_edge(line_dict))
+
+            else:  # must be a quad
+                # we convert it into (up to) 4 lines
+                ul, ur, ll, lr = i[1]
+
+                line_dict = make_line(p, ul, ll, clip)
+                if line_dict:
+                    EDGES.append(line_to_edge(line_dict))
+
+                line_dict = make_line(p, ll, lr, clip)
+                if line_dict:
+                    EDGES.append(line_to_edge(line_dict))
+
+                line_dict = make_line(p, lr, ur, clip)
+                if line_dict:
+                    EDGES.append(line_to_edge(line_dict))
+
+                line_dict = make_line(p, ur, ul, clip)
+                if line_dict:
+                    EDGES.append(line_to_edge(line_dict))
+
+    path = {"color": (0, 0, 0), "fill": None, "width": 1}
+    for bbox in bboxes:  # add the border lines for all enveloping bboxes
+        line_dict = make_line(path, bbox.tl, bbox.tr, clip)
+        if line_dict:
+            EDGES.append(line_to_edge(line_dict))
+
+        line_dict = make_line(path, bbox.bl, bbox.br, clip)
+        if line_dict:
+            EDGES.append(line_to_edge(line_dict))
+
+        line_dict = make_line(path, bbox.tl, bbox.bl, clip)
+        if line_dict:
+            EDGES.append(line_to_edge(line_dict))
+
+        line_dict = make_line(path, bbox.tr, bbox.br, clip)
+        if line_dict:
+            EDGES.append(line_to_edge(line_dict))
+
+    if add_lines is not None:  # add user-specified lines
+        assert isinstance(add_lines, (tuple, list))
+    else:
+        add_lines = []
+    for p1, p2 in add_lines:
+        p1 = Point(p1)
+        p2 = Point(p2)
+        line_dict = make_line(path, p1, p2, clip)
+        if line_dict:
+            EDGES.append(line_to_edge(line_dict))
+
+    if add_boxes is not None:  # add user-specified rectangles
+        assert isinstance(add_boxes, (tuple, list))
+    else:
+        add_boxes = []
+    for box in add_boxes:
+        r = Rect(box)
+        line_dict = make_line(path, r.tl, r.bl, clip)
+        if line_dict:
+            EDGES.append(line_to_edge(line_dict))
+        line_dict = make_line(path, r.bl, r.br, clip)
+        if line_dict:
+            EDGES.append(line_to_edge(line_dict))
+        line_dict = make_line(path, r.br, r.tr, clip)
+        if line_dict:
+            EDGES.append(line_to_edge(line_dict))
+        line_dict = make_line(path, r.tr, r.tl, clip)
+        if line_dict:
+            EDGES.append(line_to_edge(line_dict))
+
+
+def page_rotation_set0(page):
+    """Nullify page rotation.
+
+    To correctly detect tables, page rotation must be zero.
+    This function performs the necessary adjustments and returns information
+    for reverting this changes.
+    """
+    mediabox = page.mediabox
+    rot = page.rotation  # contains normalized rotation value
+    # need to derotate the page's content
+    mb = page.mediabox  # current mediabox
+
+    if rot == 90:
+        # before derotation, shift content horizontally
+        mat0 = Matrix(1, 0, 0, 1, mb.y1 - mb.x1 - mb.x0 - mb.y0, 0)
+    elif rot == 270:
+        # before derotation, shift content vertically
+        mat0 = Matrix(1, 0, 0, 1, 0, mb.x1 - mb.y1 - mb.y0 - mb.x0)
+    else:
+        mat0 = Matrix(1, 0, 0, 1, -2 * mb.x0, -2 * mb.y0)
+
+    # prefix with derotation matrix
+    mat = mat0 * page.derotation_matrix
+    cmd = b"%g %g %g %g %g %g cm " % tuple(mat)
+    xref = TOOLS._insert_contents(page, cmd, 0)
+
+    # swap x- and y-coordinates
+    if rot in (90, 270):
+        x0, y0, x1, y1 = mb
+        mb.x0 = y0
+        mb.y0 = x0
+        mb.x1 = y1
+        mb.y1 = x1
+        page.set_mediabox(mb)
+
+    page.set_rotation(0)
+
+    # refresh the page to apply these changes
+    doc = page.parent
+    pno = page.number
+    page = doc[pno]
+    return page, xref, rot, mediabox
+
+
+def page_rotation_reset(page, xref, rot, mediabox):
+    """Reset page rotation to original values.
+
+    To be used before we return tables."""
+    doc = page.parent  # document of the page
+    doc.update_stream(xref, b" ")  # remove de-rotation matrix
+    page.set_mediabox(mediabox)  # set mediabox to old value
+    page.set_rotation(rot)  # set rotation to old value
+    pno = page.number
+    page = doc[pno]  # update page info
+    return page
+
+
+def find_tables(
+    page,
+    clip=None,
+    vertical_strategy: str = "lines",
+    horizontal_strategy: str = "lines",
+    vertical_lines: list = None,
+    horizontal_lines: list = None,
+    snap_tolerance: float = DEFAULT_SNAP_TOLERANCE,
+    snap_x_tolerance: float = None,
+    snap_y_tolerance: float = None,
+    join_tolerance: float = DEFAULT_JOIN_TOLERANCE,
+    join_x_tolerance: float = None,
+    join_y_tolerance: float = None,
+    edge_min_length: float = 3,
+    min_words_vertical: float = DEFAULT_MIN_WORDS_VERTICAL,
+    min_words_horizontal: float = DEFAULT_MIN_WORDS_HORIZONTAL,
+    intersection_tolerance: float = 3,
+    intersection_x_tolerance: float = None,
+    intersection_y_tolerance: float = None,
+    text_tolerance=3,
+    text_x_tolerance=3,
+    text_y_tolerance=3,
+    strategy=None,  # offer abbreviation
+    add_lines=None,  # user-specified lines
+    add_boxes=None,  # user-specified rectangles
+    paths=None,  # accept vector graphics as parameter
+):
+    global CHARS, EDGES
+    CHARS = []
+    EDGES = []
+    old_small = bool(TOOLS.set_small_glyph_heights())  # save old value
+    TOOLS.set_small_glyph_heights(True)  # we need minimum bboxes
+    if page.rotation != 0:
+        page, old_xref, old_rot, old_mediabox = page_rotation_set0(page)
+    else:
+        old_xref, old_rot, old_mediabox = None, None, None
+
+    if snap_x_tolerance is None:
+        snap_x_tolerance = UNSET
+    if snap_y_tolerance is None:
+        snap_y_tolerance = UNSET
+    if join_x_tolerance is None:
+        join_x_tolerance = UNSET
+    if join_y_tolerance is None:
+        join_y_tolerance = UNSET
+    if intersection_x_tolerance is None:
+        intersection_x_tolerance = UNSET
+    if intersection_y_tolerance is None:
+        intersection_y_tolerance = UNSET
+    if strategy is not None:
+        vertical_strategy = strategy
+        horizontal_strategy = strategy
+
+    settings = {
+        "vertical_strategy": vertical_strategy,
+        "horizontal_strategy": horizontal_strategy,
+        "explicit_vertical_lines": vertical_lines,
+        "explicit_horizontal_lines": horizontal_lines,
+        "snap_tolerance": snap_tolerance,
+        "snap_x_tolerance": snap_x_tolerance,
+        "snap_y_tolerance": snap_y_tolerance,
+        "join_tolerance": join_tolerance,
+        "join_x_tolerance": join_x_tolerance,
+        "join_y_tolerance": join_y_tolerance,
+        "edge_min_length": edge_min_length,
+        "min_words_vertical": min_words_vertical,
+        "min_words_horizontal": min_words_horizontal,
+        "intersection_tolerance": intersection_tolerance,
+        "intersection_x_tolerance": intersection_x_tolerance,
+        "intersection_y_tolerance": intersection_y_tolerance,
+        "text_tolerance": text_tolerance,
+        "text_x_tolerance": text_x_tolerance,
+        "text_y_tolerance": text_y_tolerance,
+    }
+    tset = TableSettings.resolve(settings=settings)
+    page.table_settings = tset
+
+    make_chars(page, clip=clip)  # create character list of page
+    make_edges(
+        page,
+        clip=clip,
+        tset=tset,
+        paths=paths,
+        add_lines=add_lines,
+        add_boxes=add_boxes,
+    )  # create lines and curves
+    tables = TableFinder(page, settings=tset)
+
+    TOOLS.set_small_glyph_heights(old_small)
+    if old_xref is not None:
+        page = page_rotation_reset(page, old_xref, old_rot, old_mediabox)
+    return tables
diff --git a/lib/python3.1/site-packages/pymupdf/utils.py b/lib/python3.1/site-packages/pymupdf/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..f4f45a0b97c289f7e3c44180c1f0eff9de2f0199
--- /dev/null
+++ b/lib/python3.1/site-packages/pymupdf/utils.py
@@ -0,0 +1,5675 @@
+# ------------------------------------------------------------------------
+# Copyright 2020-2022, Harald Lieder, mailto:harald.lieder@outlook.com
+# License: GNU AFFERO GPL 3.0, https://www.gnu.org/licenses/agpl-3.0.html
+#
+# Part of "PyMuPDF", a Python binding for "MuPDF" (http://mupdf.com), a
+# lightweight PDF, XPS, and E-book viewer, renderer and toolkit which is
+# maintained and developed by Artifex Software, Inc. https://artifex.com.
+# ------------------------------------------------------------------------
+import io
+import math
+import os
+import typing
+import weakref
+
+try:
+    from . import pymupdf
+except Exception:
+    import pymupdf
+try:
+    from . import mupdf
+except Exception:
+    import mupdf
+
+_format_g = pymupdf.format_g
+
+g_exceptions_verbose = pymupdf.g_exceptions_verbose
+
+point_like = "point_like"
+rect_like = "rect_like"
+matrix_like = "matrix_like"
+quad_like = "quad_like"
+
+# ByteString is gone from typing in 3.14.
+# collections.abc.Buffer available from 3.12 only
+try:
+    ByteString = typing.ByteString
+except AttributeError:
+    # pylint: disable=unsupported-binary-operation
+    ByteString = bytes | bytearray | memoryview
+
+AnyType = typing.Any
+OptInt = typing.Union[int, None]
+OptFloat = typing.Optional[float]
+OptStr = typing.Optional[str]
+OptDict = typing.Optional[dict]
+OptBytes = typing.Optional[ByteString]
+OptSeq = typing.Optional[typing.Sequence]
+
+"""
+This is a collection of functions to extend PyMupdf.
+"""
+
+
+def write_text(
+        page: pymupdf.Page,
+        rect=None,
+        writers=None,
+        overlay=True,
+        color=None,
+        opacity=None,
+        keep_proportion=True,
+        rotate=0,
+        oc=0,
+        ) -> None:
+    """Write the text of one or more pymupdf.TextWriter objects.
+
+    Args:
+        rect: target rectangle. If None, the union of the text writers is used.
+        writers: one or more pymupdf.TextWriter objects.
+        overlay: put in foreground or background.
+        keep_proportion: maintain aspect ratio of rectangle sides.
+        rotate: arbitrary rotation angle.
+        oc: the xref of an optional content object
+    """
+    assert isinstance(page, pymupdf.Page)
+    if not writers:
+        raise ValueError("need at least one pymupdf.TextWriter")
+    if type(writers) is pymupdf.TextWriter:
+        if rotate == 0 and rect is None:
+            writers.write_text(page, opacity=opacity, color=color, overlay=overlay)
+            return None
+        else:
+            writers = (writers,)
+    clip = writers[0].text_rect
+    textdoc = pymupdf.Document()
+    tpage = textdoc.new_page(width=page.rect.width, height=page.rect.height)
+    for writer in writers:
+        clip |= writer.text_rect
+        writer.write_text(tpage, opacity=opacity, color=color)
+    if rect is None:
+        rect = clip
+    page.show_pdf_page(
+        rect,
+        textdoc,
+        0,
+        overlay=overlay,
+        keep_proportion=keep_proportion,
+        rotate=rotate,
+        clip=clip,
+        oc=oc,
+    )
+    textdoc = None
+    tpage = None
+
+
+def show_pdf_page(
+        page,
+        rect,
+        docsrc,
+        pno=0,
+        keep_proportion=True,
+        overlay=True,
+        oc=0,
+        rotate=0,
+        clip=None,
+        ) -> int:
+    """Show page number 'pno' of PDF 'docsrc' in rectangle 'rect'.
+
+    Args:
+        rect: (rect-like) where to place the source image
+        docsrc: (document) source PDF
+        pno: (int) source page number
+        keep_proportion: (bool) do not change width-height-ratio
+        overlay: (bool) put in foreground
+        oc: (xref) make visibility dependent on this OCG / OCMD (which must be defined in the target PDF)
+        rotate: (int) degrees (multiple of 90)
+        clip: (rect-like) part of source page rectangle
+    Returns:
+        xref of inserted object (for reuse)
+    """
+    def calc_matrix(sr, tr, keep=True, rotate=0):
+        """Calculate transformation matrix from source to target rect.
+
+        Notes:
+            The product of four matrices in this sequence: (1) translate correct
+            source corner to origin, (2) rotate, (3) scale, (4) translate to
+            target's top-left corner.
+        Args:
+            sr: source rect in PDF (!) coordinate system
+            tr: target rect in PDF coordinate system
+            keep: whether to keep source ratio of width to height
+            rotate: rotation angle in degrees
+        Returns:
+            Transformation matrix.
+        """
+        # calc center point of source rect
+        smp = (sr.tl + sr.br) / 2.0
+        # calc center point of target rect
+        tmp = (tr.tl + tr.br) / 2.0
+
+        # m moves to (0, 0), then rotates
+        m = pymupdf.Matrix(1, 0, 0, 1, -smp.x, -smp.y) * pymupdf.Matrix(rotate)
+
+        sr1 = sr * m  # resulting source rect to calculate scale factors
+
+        fw = tr.width / sr1.width  # scale the width
+        fh = tr.height / sr1.height  # scale the height
+        if keep:
+            fw = fh = min(fw, fh)  # take min if keeping aspect ratio
+
+        m *= pymupdf.Matrix(fw, fh)  # concat scale matrix
+        m *= pymupdf.Matrix(1, 0, 0, 1, tmp.x, tmp.y)  # concat move to target center
+        return pymupdf.JM_TUPLE(m)
+
+    pymupdf.CheckParent(page)
+    doc = page.parent
+
+    if not doc.is_pdf or not docsrc.is_pdf:
+        raise ValueError("is no PDF")
+
+    if rect.is_empty or rect.is_infinite:
+        raise ValueError("rect must be finite and not empty")
+
+    while pno < 0:  # support negative page numbers
+        pno += docsrc.page_count
+    src_page = docsrc[pno]  # load source page
+
+    tar_rect = rect * ~page.transformation_matrix  # target rect in PDF coordinates
+
+    src_rect = src_page.rect if not clip else src_page.rect & clip  # source rect
+    if src_rect.is_empty or src_rect.is_infinite:
+        raise ValueError("clip must be finite and not empty")
+    src_rect = src_rect * ~src_page.transformation_matrix  # ... in PDF coord
+
+    matrix = calc_matrix(src_rect, tar_rect, keep=keep_proportion, rotate=rotate)
+
+    # list of existing /Form /XObjects
+    ilst = [i[1] for i in doc.get_page_xobjects(page.number)]
+    ilst += [i[7] for i in doc.get_page_images(page.number)]
+    ilst += [i[4] for i in doc.get_page_fonts(page.number)]
+
+    # create a name not in that list
+    n = "fzFrm"
+    i = 0
+    _imgname = n + "0"
+    while _imgname in ilst:
+        i += 1
+        _imgname = n + str(i)
+
+    isrc = docsrc._graft_id  # used as key for graftmaps
+    if doc._graft_id == isrc:
+        raise ValueError("source document must not equal target")
+
+    # retrieve / make pymupdf.Graftmap for source PDF
+    gmap = doc.Graftmaps.get(isrc, None)
+    if gmap is None:
+        gmap = pymupdf.Graftmap(doc)
+        doc.Graftmaps[isrc] = gmap
+
+    # take note of generated xref for automatic reuse
+    pno_id = (isrc, pno)  # id of docsrc[pno]
+    xref = doc.ShownPages.get(pno_id, 0)
+
+    if overlay:
+        page.wrap_contents()  # ensure a balanced graphics state
+    xref = page._show_pdf_page(
+        src_page,
+        overlay=overlay,
+        matrix=matrix,
+        xref=xref,
+        oc=oc,
+        clip=src_rect,
+        graftmap=gmap,
+        _imgname=_imgname,
+    )
+    doc.ShownPages[pno_id] = xref
+
+    return xref
+
+
+def replace_image(page: pymupdf.Page, xref: int, *, filename=None, pixmap=None, stream=None):
+    """Replace the image referred to by xref.
+
+    Replace the image by changing the object definition stored under xref. This
+    will leave the pages appearance instructions intact, so the new image is
+    being displayed with the same bbox, rotation etc.
+    By providing a small fully transparent image, an effect as if the image had
+    been deleted can be achieved.
+    A typical use may include replacing large images by a smaller version,
+    e.g. with a lower resolution or graylevel instead of colored.
+
+    Args:
+        xref: the xref of the image to replace.
+        filename, pixmap, stream: exactly one of these must be provided. The
+            meaning being the same as in Page.insert_image.
+    """
+    doc = page.parent  # the owning document
+    if not doc.xref_is_image(xref):
+        raise ValueError("xref not an image")  # insert new image anywhere in page
+    if bool(filename) + bool(stream) + bool(pixmap) != 1:
+        raise ValueError("Exactly one of filename/stream/pixmap must be given")
+    new_xref = page.insert_image(
+        page.rect, filename=filename, stream=stream, pixmap=pixmap
+    )
+    doc.xref_copy(new_xref, xref)  # copy over new to old
+    last_contents_xref = page.get_contents()[-1]
+    # new image insertion has created a new /Contents source,
+    # which we will set to spaces now
+    doc.update_stream(last_contents_xref, b" ")
+    page._image_info = None  # clear cache of extracted image information
+
+
+def delete_image(page: pymupdf.Page, xref: int):
+    """Delete the image referred to by xef.
+
+    Actually replaces by a small transparent Pixmap using method Page.replace_image.
+
+    Args:
+        xref: xref of the image to delete.
+    """
+    # make a small 100% transparent pixmap (of just any dimension)
+    pix = pymupdf.Pixmap(pymupdf.csGRAY, (0, 0, 1, 1), 1)
+    pix.clear_with()  # clear all samples bytes to 0x00
+    page.replace_image(xref, pixmap=pix)
+
+
+def insert_image(
+        page,
+        rect,
+        *,
+        alpha=-1,
+        filename=None,
+        height=0,
+        keep_proportion=True,
+        mask=None,
+        oc=0,
+        overlay=True,
+        pixmap=None,
+        rotate=0,
+        stream=None,
+        width=0,
+        xref=0,
+        ):
+    """Insert an image for display in a rectangle.
+
+    Args:
+        rect: (rect_like) position of image on the page.
+        alpha: (int, optional) set to 0 if image has no transparency.
+        filename: (str, Path, file object) image filename.
+        height: (int)
+        keep_proportion: (bool) keep width / height ratio (default).
+        mask: (bytes, optional) image consisting of alpha values to use.
+        oc: (int) xref of OCG or OCMD to declare as Optional Content.
+        overlay: (bool) put in foreground (default) or background.
+        pixmap: (pymupdf.Pixmap) use this as image.
+        rotate: (int) rotate by 0, 90, 180 or 270 degrees.
+        stream: (bytes) use this as image.
+        width: (int)
+        xref: (int) use this as image.
+
+    'page' and 'rect' are positional, all other parameters are keywords.
+
+    If 'xref' is given, that image is used. Other input options are ignored.
+    Else, exactly one of pixmap, stream or filename must be given.
+
+    'alpha=0' for non-transparent images improves performance significantly.
+    Affects stream and filename only.
+
+    Optimum transparent insertions are possible by using filename / stream in
+    conjunction with a 'mask' image of alpha values.
+
+    Returns:
+        xref (int) of inserted image. Re-use as argument for multiple insertions.
+    """
+    pymupdf.CheckParent(page)
+    doc = page.parent
+    if not doc.is_pdf:
+        raise ValueError("is no PDF")
+
+    if xref == 0 and (bool(filename) + bool(stream) + bool(pixmap) != 1):
+        raise ValueError("xref=0 needs exactly one of filename, pixmap, stream")
+
+    if filename:
+        if type(filename) is str:
+            pass
+        elif hasattr(filename, "absolute"):
+            filename = str(filename)
+        elif hasattr(filename, "name"):
+            filename = filename.name
+        else:
+            raise ValueError("bad filename")
+
+    if filename and not os.path.exists(filename):
+        raise FileNotFoundError("No such file: '%s'" % filename)
+    elif stream and type(stream) not in (bytes, bytearray, io.BytesIO):
+        raise ValueError("stream must be bytes-like / BytesIO")
+    elif pixmap and type(pixmap) is not pymupdf.Pixmap:
+        raise ValueError("pixmap must be a pymupdf.Pixmap")
+    if mask and not (stream or filename):
+        raise ValueError("mask requires stream or filename")
+    if mask and type(mask) not in (bytes, bytearray, io.BytesIO):
+        raise ValueError("mask must be bytes-like / BytesIO")
+    while rotate < 0:
+        rotate += 360
+    while rotate >= 360:
+        rotate -= 360
+    if rotate not in (0, 90, 180, 270):
+        raise ValueError("bad rotate value")
+
+    r = pymupdf.Rect(rect)
+    if r.is_empty or r.is_infinite:
+        raise ValueError("rect must be finite and not empty")
+    clip = r * ~page.transformation_matrix
+
+    # Create a unique image reference name.
+    ilst = [i[7] for i in doc.get_page_images(page.number)]
+    ilst += [i[1] for i in doc.get_page_xobjects(page.number)]
+    ilst += [i[4] for i in doc.get_page_fonts(page.number)]
+    n = "fzImg"  # 'pymupdf image'
+    i = 0
+    _imgname = n + "0"  # first name candidate
+    while _imgname in ilst:
+        i += 1
+        _imgname = n + str(i)  # try new name
+
+    if overlay:
+        page.wrap_contents()  # ensure a balanced graphics state
+    digests = doc.InsertedImages
+    xref, digests = page._insert_image(
+        filename=filename,
+        pixmap=pixmap,
+        stream=stream,
+        imask=mask,
+        clip=clip,
+        overlay=overlay,
+        oc=oc,
+        xref=xref,
+        rotate=rotate,
+        keep_proportion=keep_proportion,
+        width=width,
+        height=height,
+        alpha=alpha,
+        _imgname=_imgname,
+        digests=digests,
+    )
+    if digests is not None:
+        doc.InsertedImages = digests
+
+    return xref
+
+
+def search_for(
+        page,
+        text,
+        *,
+        clip=None,
+        quads=False,
+        flags=pymupdf.TEXT_DEHYPHENATE
+            | pymupdf.TEXT_PRESERVE_WHITESPACE
+            | pymupdf.TEXT_PRESERVE_LIGATURES
+            | pymupdf.TEXT_MEDIABOX_CLIP
+            ,
+        textpage=None,
+        ) -> list:
+    """Search for a string on a page.
+
+    Args:
+        text: string to be searched for
+        clip: restrict search to this rectangle
+        quads: (bool) return quads instead of rectangles
+        flags: bit switches, default: join hyphened words
+        textpage: a pre-created pymupdf.TextPage
+    Returns:
+        a list of rectangles or quads, each containing one occurrence.
+    """
+    if clip is not None:
+        clip = pymupdf.Rect(clip)
+
+    pymupdf.CheckParent(page)
+    tp = textpage
+    if tp is None:
+        tp = page.get_textpage(clip=clip, flags=flags)  # create pymupdf.TextPage
+    elif getattr(tp, "parent") != page:
+        raise ValueError("not a textpage of this page")
+    rlist = tp.search(text, quads=quads)
+    if textpage is None:
+        del tp
+    return rlist
+
+
+def search_page_for(
+    doc: pymupdf.Document,
+    pno: int,
+    text: str,
+    quads: bool = False,
+    clip: rect_like = None,
+    flags: int = pymupdf.TEXT_DEHYPHENATE
+            | pymupdf.TEXT_PRESERVE_LIGATURES
+            | pymupdf.TEXT_PRESERVE_WHITESPACE
+            | pymupdf.TEXT_MEDIABOX_CLIP
+            ,
+    textpage: pymupdf.TextPage = None,
+) -> list:
+    """Search for a string on a page.
+
+    Args:
+        pno: page number
+        text: string to be searched for
+        clip: restrict search to this rectangle
+        quads: (bool) return quads instead of rectangles
+        flags: bit switches, default: join hyphened words
+        textpage: reuse a prepared textpage
+    Returns:
+        a list of rectangles or quads, each containing an occurrence.
+    """
+
+    return doc[pno].search_for(
+        text,
+        quads=quads,
+        clip=clip,
+        flags=flags,
+        textpage=textpage,
+    )
+
+
+def get_text_blocks(
+    page: pymupdf.Page,
+    clip: rect_like = None,
+    flags: OptInt = None,
+    textpage: pymupdf.TextPage = None,
+    sort: bool = False,
+) -> list:
+    """Return the text blocks on a page.
+
+    Notes:
+        Lines in a block are concatenated with line breaks.
+    Args:
+        flags: (int) control the amount of data parsed into the textpage.
+    Returns:
+        A list of the blocks. Each item contains the containing rectangle
+        coordinates, text lines, running block number and block type.
+    """
+    pymupdf.CheckParent(page)
+    if flags is None:
+        flags = pymupdf.TEXTFLAGS_BLOCKS
+    tp = textpage
+    if tp is None:
+        tp = page.get_textpage(clip=clip, flags=flags)
+    elif getattr(tp, "parent") != page:
+        raise ValueError("not a textpage of this page")
+
+    blocks = tp.extractBLOCKS()
+    if textpage is None:
+        del tp
+    if sort:
+        blocks.sort(key=lambda b: (b[3], b[0]))
+    return blocks
+
+
+def get_text_words(
+    page: pymupdf.Page,
+    clip: rect_like = None,
+    flags: OptInt = None,
+    textpage: pymupdf.TextPage = None,
+    sort: bool = False,
+    delimiters=None,
+    tolerance=3,
+) -> list:
+    """Return the text words as a list with the bbox for each word.
+
+    Args:
+        page: pymupdf.Page
+        clip: (rect-like) area on page to consider
+        flags: (int) control the amount of data parsed into the textpage.
+        textpage: (pymupdf.TextPage) either passed-in or None.
+        sort: (bool) sort the words in reading sequence.
+        delimiters: (str,list) characters to use as word delimiters.
+        tolerance: (float) consider words to be part of the same line if
+            top or bottom coordinate are not larger than this. Relevant
+            only if sort=True.
+
+    Returns:
+        Word tuples (x0, y0, x1, y1, "word", bno, lno, wno).
+    """
+
+    def sort_words(words):
+        """Sort words line-wise, forgiving small deviations."""
+        words.sort(key=lambda w: (w[3], w[0]))
+        nwords = []  # final word list
+        line = [words[0]]  # collects words roughly in same line
+        lrect = pymupdf.Rect(words[0][:4])  # start the line rectangle
+        for w in words[1:]:
+            wrect = pymupdf.Rect(w[:4])
+            if (
+                abs(wrect.y0 - lrect.y0) <= tolerance
+                or abs(wrect.y1 - lrect.y1) <= tolerance
+            ):
+                line.append(w)
+                lrect |= wrect
+            else:
+                line.sort(key=lambda w: w[0])  # sort words in line l-t-r
+                nwords.extend(line)  # append to final words list
+                line = [w]  # start next line
+                lrect = wrect  # start next line rect
+
+        line.sort(key=lambda w: w[0])  # sort words in line l-t-r
+        nwords.extend(line)  # append to final words list
+
+        return nwords
+
+    pymupdf.CheckParent(page)
+    if flags is None:
+        flags = pymupdf.TEXTFLAGS_WORDS
+    tp = textpage
+    if tp is None:
+        tp = page.get_textpage(clip=clip, flags=flags)
+    elif getattr(tp, "parent") != page:
+        raise ValueError("not a textpage of this page")
+
+    words = tp.extractWORDS(delimiters)
+
+    # if textpage was given, we subselect the words in clip
+    if textpage is not None and clip is not None:
+        # sub-select words contained in clip
+        clip = pymupdf.Rect(clip)
+        words = [
+            w for w in words if abs(clip & w[:4]) >= 0.5 * abs(pymupdf.Rect(w[:4]))
+        ]
+
+    if textpage is None:
+        del tp
+    if words and sort:
+        # advanced sort if any words found
+        words = sort_words(words)
+
+    return words
+
+
+def get_sorted_text(
+    page: pymupdf.Page,
+    clip: rect_like = None,
+    flags: OptInt = None,
+    textpage: pymupdf.TextPage = None,
+    tolerance=3,
+) -> str:
+    """Extract plain text avoiding unacceptable line breaks.
+
+    Text contained in clip will be sorted in reading sequence. Some effort
+    is also spent to simulate layout vertically and horizontally.
+
+    Args:
+        page: pymupdf.Page
+        clip: (rect-like) only consider text inside
+        flags: (int) text extraction flags
+        textpage: pymupdf.TextPage
+        tolerance: (float) consider words to be on the same line if their top
+            or bottom coordinates do not differ more than this.
+
+    Notes:
+        If a TextPage is provided, all text is checked for being inside clip
+        with at least 50% of its bbox.
+        This allows to use some "global" TextPage in conjunction with sub-
+        selecting words in parts of the defined TextPage rectangle.
+
+    Returns:
+        A text string in reading sequence. Left indentation of each line,
+        inter-line and inter-word distances strive to reflect the layout.
+    """
+
+    def line_text(clip, line):
+        """Create the string of one text line.
+
+        We are trying to simulate some horizontal layout here, too.
+
+        Args:
+            clip: (pymupdf.Rect) the area from which all text is being read.
+            line: (list) word tuples (rect, text) contained in the line
+        Returns:
+            Text in this line. Generated from words in 'line'. Distance from
+            predecessor is translated to multiple spaces, thus simulating
+            text indentations and large horizontal distances.
+        """
+        line.sort(key=lambda w: w[0].x0)
+        ltext = ""  # text in the line
+        x1 = clip.x0  # end coordinate of ltext
+        lrect = pymupdf.EMPTY_RECT()  # bbox of this line
+        for r, t in line:
+            lrect |= r  # update line bbox
+            # convert distance to previous word to multiple spaces
+            dist = max(
+                int(round((r.x0 - x1) / r.width * len(t))),
+                0 if (x1 == clip.x0 or r.x0 <= x1) else 1,
+            )  # number of space characters
+
+            ltext += " " * dist + t  # append word string
+            x1 = r.x1  # update new end position
+        return ltext
+
+    # Extract words in correct sequence first.
+    words = [
+        (pymupdf.Rect(w[:4]), w[4])
+        for w in get_text_words(
+            page,
+            clip=clip,
+            flags=flags,
+            textpage=textpage,
+            sort=True,
+            tolerance=tolerance,
+        )
+    ]
+
+    if not words:  # no text present
+        return ""
+    totalbox = pymupdf.EMPTY_RECT()  # area covering all text
+    for wr, text in words:
+        totalbox |= wr
+
+    lines = []  # list of reconstituted lines
+    line = [words[0]]  # current line
+    lrect = words[0][0]  # the line's rectangle
+
+    # walk through the words
+    for wr, text in words[1:]:  # start with second word
+        w0r, _ = line[-1]  # read previous word in current line
+
+        # if this word matches top or bottom of the line, append it
+        if abs(lrect.y0 - wr.y0) <= tolerance or abs(lrect.y1 - wr.y1) <= tolerance:
+            line.append((wr, text))
+            lrect |= wr
+        else:
+            # output current line and re-initialize
+            ltext = line_text(totalbox, line)
+            lines.append((lrect, ltext))
+            line = [(wr, text)]
+            lrect = wr
+
+    # also append unfinished last line
+    ltext = line_text(totalbox, line)
+    lines.append((lrect, ltext))
+
+    # sort all lines vertically
+    lines.sort(key=lambda l: (l[0].y1))
+
+    text = lines[0][1]  # text of first line
+    y1 = lines[0][0].y1  # its bottom coordinate
+    for lrect, ltext in lines[1:]:
+        distance = min(int(round((lrect.y0 - y1) / lrect.height)), 5)
+        breaks = "\n" * (distance + 1)
+        text += breaks + ltext
+        y1 = lrect.y1
+
+    # return text in clip
+    return text
+
+
+def get_textbox(
+    page: pymupdf.Page,
+    rect: rect_like,
+    textpage: pymupdf.TextPage = None,
+) -> str:
+    tp = textpage
+    if tp is None:
+        tp = page.get_textpage()
+    elif getattr(tp, "parent") != page:
+        raise ValueError("not a textpage of this page")
+    rc = tp.extractTextbox(rect)
+    if textpage is None:
+        del tp
+    return rc
+
+
+def get_text_selection(
+    page: pymupdf.Page,
+    p1: point_like,
+    p2: point_like,
+    clip: rect_like = None,
+    textpage: pymupdf.TextPage = None,
+):
+    pymupdf.CheckParent(page)
+    tp = textpage
+    if tp is None:
+        tp = page.get_textpage(clip=clip, flags=pymupdf.TEXT_DEHYPHENATE)
+    elif getattr(tp, "parent") != page:
+        raise ValueError("not a textpage of this page")
+    rc = tp.extractSelection(p1, p2)
+    if textpage is None:
+        del tp
+    return rc
+
+
+def get_textpage_ocr(
+    page: pymupdf.Page,
+    flags: int = 0,
+    language: str = "eng",
+    dpi: int = 72,
+    full: bool = False,
+    tessdata: str = None,
+) -> pymupdf.TextPage:
+    """Create a Textpage from combined results of normal and OCR text parsing.
+
+    Args:
+        flags: (int) control content becoming part of the result.
+        language: (str) specify expected language(s). Default is "eng" (English).
+        dpi: (int) resolution in dpi, default 72.
+        full: (bool) whether to OCR the full page image, or only its images (default)
+    """
+    pymupdf.CheckParent(page)
+    tessdata = pymupdf.get_tessdata(tessdata)
+
+    def full_ocr(page, dpi, language, flags):
+        zoom = dpi / 72
+        mat = pymupdf.Matrix(zoom, zoom)
+        pix = page.get_pixmap(matrix=mat)
+        ocr_pdf = pymupdf.Document(
+                "pdf",
+                pix.pdfocr_tobytes(
+                    compress=False,
+                    language=language,
+                    tessdata=tessdata,
+                    ),
+                )
+        ocr_page = ocr_pdf.load_page(0)
+        unzoom = page.rect.width / ocr_page.rect.width
+        ctm = pymupdf.Matrix(unzoom, unzoom) * page.derotation_matrix
+        tpage = ocr_page.get_textpage(flags=flags, matrix=ctm)
+        ocr_pdf.close()
+        pix = None
+        tpage.parent = weakref.proxy(page)
+        return tpage
+
+    # if OCR for the full page, OCR its pixmap @ desired dpi
+    if full:
+        return full_ocr(page, dpi, language, flags)
+
+    # For partial OCR, make a normal textpage, then extend it with text that
+    # is OCRed from each image.
+    # Because of this, we need the images flag bit set ON.
+    tpage = page.get_textpage(flags=flags)
+    for block in page.get_text("dict", flags=pymupdf.TEXT_PRESERVE_IMAGES)["blocks"]:
+        if block["type"] != 1:  # only look at images
+            continue
+        bbox = pymupdf.Rect(block["bbox"])
+        if bbox.width <= 3 or bbox.height <= 3:  # ignore tiny stuff
+            continue
+        try:
+            pix = pymupdf.Pixmap(block["image"])  # get image pixmap
+            if pix.n - pix.alpha != 3:  # we need to convert this to RGB!
+                pix = pymupdf.Pixmap(pymupdf.csRGB, pix)
+            if pix.alpha:  # must remove alpha channel
+                pix = pymupdf.Pixmap(pix, 0)
+            imgdoc = pymupdf.Document(
+                    "pdf",
+                    pix.pdfocr_tobytes(language=language, tessdata=tessdata),
+                    )  # pdf with OCRed page
+            imgpage = imgdoc.load_page(0)  # read image as a page
+            pix = None
+            # compute matrix to transform coordinates back to that of 'page'
+            imgrect = imgpage.rect  # page size of image PDF
+            shrink = pymupdf.Matrix(1 / imgrect.width, 1 / imgrect.height)
+            mat = shrink * block["transform"]
+            imgpage.extend_textpage(tpage, flags=0, matrix=mat)
+            imgdoc.close()
+        except (RuntimeError, mupdf.FzErrorBase):
+            if 0 and g_exceptions_verbose:
+                # Don't show exception info here because it can happen in
+                # normal operation (see test_3842b).
+                pymupdf.exception_info()
+            tpage = None
+            pymupdf.message("Falling back to full page OCR")
+            return full_ocr(page, dpi, language, flags)
+
+    return tpage
+
+
+def get_image_info(page: pymupdf.Page, hashes: bool = False, xrefs: bool = False) -> list:
+    """Extract image information only from a pymupdf.TextPage.
+
+    Args:
+        hashes: (bool) include MD5 hash for each image.
+        xrefs: (bool) try to find the xref for each image. Sets hashes to true.
+    """
+    doc = page.parent
+    if xrefs and doc.is_pdf:
+        hashes = True
+    if not doc.is_pdf:
+        xrefs = False
+    imginfo = getattr(page, "_image_info", None)
+    if imginfo and not xrefs:
+        return imginfo
+    if not imginfo:
+        tp = page.get_textpage(flags=pymupdf.TEXT_PRESERVE_IMAGES)
+        imginfo = tp.extractIMGINFO(hashes=hashes)
+        del tp
+        if hashes:
+            page._image_info = imginfo
+    if not xrefs or not doc.is_pdf:
+        return imginfo
+    imglist = page.get_images()
+    digests = {}
+    for item in imglist:
+        xref = item[0]
+        pix = pymupdf.Pixmap(doc, xref)
+        digests[pix.digest] = xref
+        del pix
+    for i in range(len(imginfo)):
+        item = imginfo[i]
+        xref = digests.get(item["digest"], 0)
+        item["xref"] = xref
+        imginfo[i] = item
+    return imginfo
+
+
+def get_image_rects(page: pymupdf.Page, name, transform=False) -> list:
+    """Return list of image positions on a page.
+
+    Args:
+        name: (str, list, int) image identification. May be reference name, an
+              item of the page's image list or an xref.
+        transform: (bool) whether to also return the transformation matrix.
+    Returns:
+        A list of pymupdf.Rect objects or tuples of (pymupdf.Rect, pymupdf.Matrix)
+        for all image locations on the page.
+    """
+    if type(name) in (list, tuple):
+        xref = name[0]
+    elif type(name) is int:
+        xref = name
+    else:
+        imglist = [i for i in page.get_images() if i[7] == name]
+        if imglist == []:
+            raise ValueError("bad image name")
+        elif len(imglist) != 1:
+            raise ValueError("multiple image names found")
+        xref = imglist[0][0]
+    pix = pymupdf.Pixmap(page.parent, xref)  # make pixmap of the image to compute MD5
+    digest = pix.digest
+    del pix
+    infos = page.get_image_info(hashes=True)
+    if not transform:
+        bboxes = [pymupdf.Rect(im["bbox"]) for im in infos if im["digest"] == digest]
+    else:
+        bboxes = [
+            (pymupdf.Rect(im["bbox"]), pymupdf.Matrix(im["transform"]))
+            for im in infos
+            if im["digest"] == digest
+        ]
+    return bboxes
+
+
+def get_text(
+    page: pymupdf.Page,
+    option: str = "text",
+    *,
+    clip: rect_like = None,
+    flags: OptInt = None,
+    textpage: pymupdf.TextPage = None,
+    sort: bool = False,
+    delimiters=None,
+    tolerance=3,
+):
+    """Extract text from a page or an annotation.
+
+    This is a unifying wrapper for various methods of the pymupdf.TextPage class.
+
+    Args:
+        option: (str) text, words, blocks, html, dict, json, rawdict, xhtml or xml.
+        clip: (rect-like) restrict output to this area.
+        flags: bit switches to e.g. exclude images or decompose ligatures.
+        textpage: reuse this pymupdf.TextPage and make no new one. If specified,
+            'flags' and 'clip' are ignored.
+
+    Returns:
+        the output of methods get_text_words / get_text_blocks or pymupdf.TextPage
+        methods extractText, extractHTML, extractDICT, extractJSON, extractRAWDICT,
+        extractXHTML or etractXML respectively.
+        Default and misspelling choice is "text".
+    """
+    formats = {
+        "text": pymupdf.TEXTFLAGS_TEXT,
+        "html": pymupdf.TEXTFLAGS_HTML,
+        "json": pymupdf.TEXTFLAGS_DICT,
+        "rawjson": pymupdf.TEXTFLAGS_RAWDICT,
+        "xml": pymupdf.TEXTFLAGS_XML,
+        "xhtml": pymupdf.TEXTFLAGS_XHTML,
+        "dict": pymupdf.TEXTFLAGS_DICT,
+        "rawdict": pymupdf.TEXTFLAGS_RAWDICT,
+        "words": pymupdf.TEXTFLAGS_WORDS,
+        "blocks": pymupdf.TEXTFLAGS_BLOCKS,
+    }
+    option = option.lower()
+    assert option in formats
+    if option not in formats:
+        option = "text"
+    if flags is None:
+        flags = formats[option]
+
+    if option == "words":
+        return get_text_words(
+            page,
+            clip=clip,
+            flags=flags,
+            textpage=textpage,
+            sort=sort,
+            delimiters=delimiters,
+        )
+    if option == "blocks":
+        return get_text_blocks(
+            page, clip=clip, flags=flags, textpage=textpage, sort=sort
+        )
+
+    if option == "text" and sort:
+        return get_sorted_text(
+            page,
+            clip=clip,
+            flags=flags,
+            textpage=textpage,
+            tolerance=tolerance,
+        )
+
+    pymupdf.CheckParent(page)
+    cb = None
+    if option in ("html", "xml", "xhtml"):  # no clipping for MuPDF functions
+        clip = page.cropbox
+    if clip is not None:
+        clip = pymupdf.Rect(clip)
+        cb = None
+    elif type(page) is pymupdf.Page:
+        cb = page.cropbox
+    # pymupdf.TextPage with or without images
+    tp = textpage
+    #pymupdf.exception_info()
+    if tp is None:
+        tp = page.get_textpage(clip=clip, flags=flags)
+    elif getattr(tp, "parent") != page:
+        raise ValueError("not a textpage of this page")
+    #pymupdf.log( '{option=}')
+    if option == "json":
+        t = tp.extractJSON(cb=cb, sort=sort)
+    elif option == "rawjson":
+        t = tp.extractRAWJSON(cb=cb, sort=sort)
+    elif option == "dict":
+        t = tp.extractDICT(cb=cb, sort=sort)
+    elif option == "rawdict":
+        t = tp.extractRAWDICT(cb=cb, sort=sort)
+    elif option == "html":
+        t = tp.extractHTML()
+    elif option == "xml":
+        t = tp.extractXML()
+    elif option == "xhtml":
+        t = tp.extractXHTML()
+    else:
+        t = tp.extractText(sort=sort)
+
+    if textpage is None:
+        del tp
+    return t
+
+
+def get_page_text(
+    doc: pymupdf.Document,
+    pno: int,
+    option: str = "text",
+    clip: rect_like = None,
+    flags: OptInt = None,
+    textpage: pymupdf.TextPage = None,
+    sort: bool = False,
+) -> typing.Any:
+    """Extract a document page's text by page number.
+
+    Notes:
+        Convenience function calling page.get_text().
+    Args:
+        pno: page number
+        option: (str) text, words, blocks, html, dict, json, rawdict, xhtml or xml.
+    Returns:
+        output from page.TextPage().
+    """
+    return doc[pno].get_text(option, clip=clip, flags=flags, sort=sort)
+
+def get_pixmap(
+        page: pymupdf.Page,
+        *,
+        matrix: matrix_like=pymupdf.Identity,
+        dpi=None,
+        colorspace: pymupdf.Colorspace=pymupdf.csRGB,
+        clip: rect_like=None,
+        alpha: bool=False,
+        annots: bool=True,
+        ) -> pymupdf.Pixmap:
+    """Create pixmap of page.
+
+    Keyword args:
+        matrix: Matrix for transformation (default: Identity).
+        dpi: desired dots per inch. If given, matrix is ignored.
+        colorspace: (str/Colorspace) cmyk, rgb, gray - case ignored, default csRGB.
+        clip: (irect-like) restrict rendering to this area.
+        alpha: (bool) whether to include alpha channel
+        annots: (bool) whether to also render annotations
+    """
+    if dpi:
+        zoom = dpi / 72
+        matrix = pymupdf.Matrix(zoom, zoom)
+
+    if type(colorspace) is str:
+        if colorspace.upper() == "GRAY":
+            colorspace = pymupdf.csGRAY
+        elif colorspace.upper() == "CMYK":
+            colorspace = pymupdf.csCMYK
+        else:
+            colorspace = pymupdf.csRGB
+    if colorspace.n not in (1, 3, 4):
+        raise ValueError("unsupported colorspace")
+
+    dl = page.get_displaylist(annots=annots)
+    pix = dl.get_pixmap(matrix=matrix, colorspace=colorspace, alpha=alpha, clip=clip)
+    dl = None
+    if dpi:
+        pix.set_dpi(dpi, dpi)
+    return pix
+
+
+def get_page_pixmap(
+    doc: pymupdf.Document,
+    pno: int,
+    *,
+    matrix: matrix_like = pymupdf.Identity,
+    dpi=None,
+    colorspace: pymupdf.Colorspace = pymupdf.csRGB,
+    clip: rect_like = None,
+    alpha: bool = False,
+    annots: bool = True,
+) -> pymupdf.Pixmap:
+    """Create pixmap of document page by page number.
+
+    Notes:
+        Convenience function calling page.get_pixmap.
+    Args:
+        pno: (int) page number
+        matrix: pymupdf.Matrix for transformation (default: pymupdf.Identity).
+        colorspace: (str,pymupdf.Colorspace) rgb, rgb, gray - case ignored, default csRGB.
+        clip: (irect-like) restrict rendering to this area.
+        alpha: (bool) include alpha channel
+        annots: (bool) also render annotations
+    """
+    return doc[pno].get_pixmap(
+            matrix=matrix,
+            dpi=dpi, colorspace=colorspace,
+            clip=clip,
+            alpha=alpha,
+            annots=annots
+            )
+
+
+def getLinkDict(ln, document=None) -> dict:
+    if isinstance(ln, pymupdf.Outline):
+        dest = ln.destination(document)
+    elif isinstance(ln, pymupdf.Link):
+        dest = ln.dest
+    else:
+        assert 0, f'Unexpected {type(ln)=}.'
+    nl = {"kind": dest.kind, "xref": 0}
+    try:
+        if hasattr(ln, 'rect'):
+            nl["from"] = ln.rect
+    except Exception:
+        # This seems to happen quite often in PyMuPDF/tests.
+        if g_exceptions_verbose >= 2:   pymupdf.exception_info()
+        pass
+    pnt = pymupdf.Point(0, 0)
+    if dest.flags & pymupdf.LINK_FLAG_L_VALID:
+        pnt.x = dest.lt.x
+    if dest.flags & pymupdf.LINK_FLAG_T_VALID:
+        pnt.y = dest.lt.y
+
+    if dest.kind == pymupdf.LINK_URI:
+        nl["uri"] = dest.uri
+
+    elif dest.kind == pymupdf.LINK_GOTO:
+        nl["page"] = dest.page
+        nl["to"] = pnt
+        if dest.flags & pymupdf.LINK_FLAG_R_IS_ZOOM:
+            nl["zoom"] = dest.rb.x
+        else:
+            nl["zoom"] = 0.0
+
+    elif dest.kind == pymupdf.LINK_GOTOR:
+        nl["file"] = dest.file_spec.replace("\\", "/")
+        nl["page"] = dest.page
+        if dest.page < 0:
+            nl["to"] = dest.dest
+        else:
+            nl["to"] = pnt
+            if dest.flags & pymupdf.LINK_FLAG_R_IS_ZOOM:
+                nl["zoom"] = dest.rb.x
+            else:
+                nl["zoom"] = 0.0
+
+    elif dest.kind == pymupdf.LINK_LAUNCH:
+        nl["file"] = dest.file_spec.replace("\\", "/")
+
+    elif dest.kind == pymupdf.LINK_NAMED:
+        # The dicts should not have same key(s).
+        assert not (dest.named.keys() & nl.keys())
+        nl.update(dest.named)
+        if 'to' in nl:
+            nl['to'] = pymupdf.Point(nl['to'])
+
+    else:
+        nl["page"] = dest.page
+    return nl
+
+
+def get_links(page: pymupdf.Page) -> list:
+    """Create a list of all links contained in a PDF page.
+
+    Notes:
+        see PyMuPDF ducmentation for details.
+    """
+
+    pymupdf.CheckParent(page)
+    ln = page.first_link
+    links = []
+    while ln:
+        nl = getLinkDict(ln, page.parent)
+        links.append(nl)
+        ln = ln.next
+    if links != [] and page.parent.is_pdf:
+        linkxrefs = [x for x in
+                #page.annot_xrefs()
+                pymupdf.JM_get_annot_xref_list2(page)
+                if x[1] == pymupdf.PDF_ANNOT_LINK  # pylint: disable=no-member
+                ]
+        if len(linkxrefs) == len(links):
+            for i in range(len(linkxrefs)):
+                links[i]["xref"] = linkxrefs[i][0]
+                links[i]["id"] = linkxrefs[i][2]
+    return links
+
+
+def get_toc(
+    doc: pymupdf.Document,
+    simple: bool = True,
+) -> list:
+    """Create a table of contents.
+
+    Args:
+        simple: a bool to control output. Returns a list, where each entry consists of outline level, title, page number and link destination (if simple = False). For details see PyMuPDF's documentation.
+    """
+    def recurse(olItem, liste, lvl):
+        """Recursively follow the outline item chain and record item information in a list."""
+        while olItem and olItem.this.m_internal:
+            if olItem.title:
+                title = olItem.title
+            else:
+                title = " "
+
+            if not olItem.is_external:
+                if olItem.uri:
+                    if olItem.page == -1:
+                        resolve = doc.resolve_link(olItem.uri)
+                        page = resolve[0] + 1
+                    else:
+                        page = olItem.page + 1
+                else:
+                    page = -1
+            else:
+                page = -1
+
+            if not simple:
+                link = getLinkDict(olItem, doc)
+                liste.append([lvl, title, page, link])
+            else:
+                liste.append([lvl, title, page])
+
+            if olItem.down:
+                liste = recurse(olItem.down, liste, lvl + 1)
+            olItem = olItem.next
+        return liste
+
+    # ensure document is open
+    if doc.is_closed:
+        raise ValueError("document closed")
+    doc.init_doc()
+    olItem = doc.outline
+    if not olItem:
+        return []
+    lvl = 1
+    liste = []
+    toc = recurse(olItem, liste, lvl)
+    if doc.is_pdf and not simple:
+        doc._extend_toc_items(toc)
+    return toc
+
+
+def del_toc_item(
+    doc: pymupdf.Document,
+    idx: int,
+) -> None:
+    """Delete TOC / bookmark item by index."""
+    xref = doc.get_outline_xrefs()[idx]
+    doc._remove_toc_item(xref)
+
+
+def set_toc_item(
+    doc: pymupdf.Document,
+    idx: int,
+    dest_dict: OptDict = None,
+    kind: OptInt = None,
+    pno: OptInt = None,
+    uri: OptStr = None,
+    title: OptStr = None,
+    to: point_like = None,
+    filename: OptStr = None,
+    zoom: float = 0,
+) -> None:
+    """Update TOC item by index.
+
+    It allows changing the item's title and link destination.
+
+    Args:
+        idx:
+            (int) desired index of the TOC list, as created by get_toc.
+        dest_dict:
+            (dict) destination dictionary as created by get_toc(False).
+            Outrules all other parameters. If None, the remaining parameters
+            are used to make a dest dictionary.
+        kind:
+            (int) kind of link (pymupdf.LINK_GOTO, etc.). If None, then only
+            the title will be updated. If pymupdf.LINK_NONE, the TOC item will
+            be deleted.
+        pno:
+            (int) page number (1-based like in get_toc). Required if
+            pymupdf.LINK_GOTO.
+        uri:
+            (str) the URL, required if pymupdf.LINK_URI.
+        title:
+            (str) the new title. No change if None.
+        to:
+            (point-like) destination on the target page. If omitted, (72, 36)
+            will be used as target coordinates.
+        filename:
+            (str) destination filename, required for pymupdf.LINK_GOTOR and
+            pymupdf.LINK_LAUNCH.
+        name:
+            (str) a destination name for pymupdf.LINK_NAMED.
+        zoom:
+            (float) a zoom factor for the target location (pymupdf.LINK_GOTO).
+    """
+    xref = doc.get_outline_xrefs()[idx]
+    page_xref = 0
+    if type(dest_dict) is dict:
+        if dest_dict["kind"] == pymupdf.LINK_GOTO:
+            pno = dest_dict["page"]
+            page_xref = doc.page_xref(pno)
+            page_height = doc.page_cropbox(pno).height
+            to = dest_dict.get('to', pymupdf.Point(72, 36))
+            to.y = page_height - to.y
+            dest_dict["to"] = to
+        action = getDestStr(page_xref, dest_dict)
+        if not action.startswith("/A"):
+            raise ValueError("bad bookmark dest")
+        color = dest_dict.get("color")
+        if color:
+            color = list(map(float, color))
+            if len(color) != 3 or min(color) < 0 or max(color) > 1:
+                raise ValueError("bad color value")
+        bold = dest_dict.get("bold", False)
+        italic = dest_dict.get("italic", False)
+        flags = italic + 2 * bold
+        collapse = dest_dict.get("collapse")
+        return doc._update_toc_item(
+            xref,
+            action=action[2:],
+            title=title,
+            color=color,
+            flags=flags,
+            collapse=collapse,
+        )
+
+    if kind == pymupdf.LINK_NONE:  # delete bookmark item
+        return doc.del_toc_item(idx)
+    if kind is None and title is None:  # treat as no-op
+        return None
+    if kind is None:  # only update title text
+        return doc._update_toc_item(xref, action=None, title=title)
+
+    if kind == pymupdf.LINK_GOTO:
+        if pno is None or pno not in range(1, doc.page_count + 1):
+            raise ValueError("bad page number")
+        page_xref = doc.page_xref(pno - 1)
+        page_height = doc.page_cropbox(pno - 1).height
+        if to is None:
+            to = pymupdf.Point(72, page_height - 36)
+        else:
+            to = pymupdf.Point(to)
+            to.y = page_height - to.y
+
+    ddict = {
+        "kind": kind,
+        "to": to,
+        "uri": uri,
+        "page": pno,
+        "file": filename,
+        "zoom": zoom,
+    }
+    action = getDestStr(page_xref, ddict)
+    if action == "" or not action.startswith("/A"):
+        raise ValueError("bad bookmark dest")
+
+    return doc._update_toc_item(xref, action=action[2:], title=title)
+
+
+def get_area(*args) -> float:
+    """Calculate area of rectangle.\nparameter is one of 'px' (default), 'in', 'cm', or 'mm'."""
+    rect = args[0]
+    if len(args) > 1:
+        unit = args[1]
+    else:
+        unit = "px"
+    u = {"px": (1, 1), "in": (1.0, 72.0), "cm": (2.54, 72.0), "mm": (25.4, 72.0)}
+    f = (u[unit][0] / u[unit][1]) ** 2
+    return f * rect.width * rect.height
+
+
+def set_metadata(doc: pymupdf.Document, m: dict = None) -> None:
+    """Update the PDF /Info object.
+
+    Args:
+        m: a dictionary like doc.metadata.
+    """
+    if not doc.is_pdf:
+        raise ValueError("is no PDF")
+    if doc.is_closed or doc.is_encrypted:
+        raise ValueError("document closed or encrypted")
+    if m is None:
+        m = {}
+    elif type(m) is not dict:
+        raise ValueError("bad metadata")
+    keymap = {
+        "author": "Author",
+        "producer": "Producer",
+        "creator": "Creator",
+        "title": "Title",
+        "format": None,
+        "encryption": None,
+        "creationDate": "CreationDate",
+        "modDate": "ModDate",
+        "subject": "Subject",
+        "keywords": "Keywords",
+        "trapped": "Trapped",
+    }
+    valid_keys = set(keymap.keys())
+    diff_set = set(m.keys()).difference(valid_keys)
+    if diff_set != set():
+        msg = "bad dict key(s): %s" % diff_set
+        raise ValueError(msg)
+
+    t, temp = doc.xref_get_key(-1, "Info")
+    if t != "xref":
+        info_xref = 0
+    else:
+        info_xref = int(temp.replace("0 R", ""))
+
+    if m == {} and info_xref == 0:  # nothing to do
+        return
+
+    if info_xref == 0:  # no prev metadata: get new xref
+        info_xref = doc.get_new_xref()
+        doc.update_object(info_xref, "<<>>")  # fill it with empty object
+        doc.xref_set_key(-1, "Info", "%i 0 R" % info_xref)
+    elif m == {}:  # remove existing metadata
+        doc.xref_set_key(-1, "Info", "null")
+        doc.init_doc()
+        return
+
+    for key, val in [(k, v) for k, v in m.items() if keymap[k] is not None]:
+        pdf_key = keymap[key]
+        if not bool(val) or val in ("none", "null"):
+            val = "null"
+        else:
+            val = pymupdf.get_pdf_str(val)
+        doc.xref_set_key(info_xref, pdf_key, val)
+    doc.init_doc()
+    return
+
+
+def getDestStr(xref: int, ddict: dict) -> str:
+    """Calculate the PDF action string.
+
+    Notes:
+        Supports Link annotations and outline items (bookmarks).
+    """
+    if not ddict:
+        return ""
+    str_goto = lambda a, b, c, d: f"/A<>"
+    str_gotor1 = lambda a, b, c, d, e, f: f"/A<>>>"
+    str_gotor2 = lambda a, b, c: f"/A<>>>"
+    str_launch = lambda a, b: f"/A<>>>"
+    str_uri = lambda a: f"/A<>"
+
+    if type(ddict) in (int, float):
+        dest = str_goto(xref, 0, ddict, 0)
+        return dest
+    d_kind = ddict.get("kind", pymupdf.LINK_NONE)
+
+    if d_kind == pymupdf.LINK_NONE:
+        return ""
+
+    if ddict["kind"] == pymupdf.LINK_GOTO:
+        d_zoom = ddict.get("zoom", 0)
+        to = ddict.get("to", pymupdf.Point(0, 0))
+        d_left, d_top = to
+        dest = str_goto(xref, d_left, d_top, d_zoom)
+        return dest
+
+    if ddict["kind"] == pymupdf.LINK_URI:
+        dest = str_uri(pymupdf.get_pdf_str(ddict["uri"]),)
+        return dest
+
+    if ddict["kind"] == pymupdf.LINK_LAUNCH:
+        fspec = pymupdf.get_pdf_str(ddict["file"])
+        dest = str_launch(fspec, fspec)
+        return dest
+
+    if ddict["kind"] == pymupdf.LINK_GOTOR and ddict["page"] < 0:
+        fspec = pymupdf.get_pdf_str(ddict["file"])
+        dest = str_gotor2(pymupdf.get_pdf_str(ddict["to"]), fspec, fspec)
+        return dest
+
+    if ddict["kind"] == pymupdf.LINK_GOTOR and ddict["page"] >= 0:
+        fspec = pymupdf.get_pdf_str(ddict["file"])
+        dest = str_gotor1(
+            ddict["page"],
+            ddict["to"].x,
+            ddict["to"].y,
+            ddict["zoom"],
+            fspec,
+            fspec,
+        )
+        return dest
+
+    return ""
+
+
+def set_toc(
+    doc: pymupdf.Document,
+    toc: list,
+    collapse: int = 1,
+) -> int:
+    """Create new outline tree (table of contents, TOC).
+
+    Args:
+        toc: (list, tuple) each entry must contain level, title, page and
+            optionally top margin on the page. None or '()' remove the TOC.
+        collapse: (int) collapses entries beyond this level. Zero or None
+            shows all entries unfolded.
+    Returns:
+        the number of inserted items, or the number of removed items respectively.
+    """
+    if doc.is_closed or doc.is_encrypted:
+        raise ValueError("document closed or encrypted")
+    if not doc.is_pdf:
+        raise ValueError("is no PDF")
+    if not toc:  # remove all entries
+        return len(doc._delToC())
+
+    # validity checks --------------------------------------------------------
+    if type(toc) not in (list, tuple):
+        raise ValueError("'toc' must be list or tuple")
+    toclen = len(toc)
+    page_count = doc.page_count
+    t0 = toc[0]
+    if type(t0) not in (list, tuple):
+        raise ValueError("items must be sequences of 3 or 4 items")
+    if t0[0] != 1:
+        raise ValueError("hierarchy level of item 0 must be 1")
+    for i in list(range(toclen - 1)):
+        t1 = toc[i]
+        t2 = toc[i + 1]
+        if not -1 <= t1[2] <= page_count:
+            raise ValueError("row %i: page number out of range" % i)
+        if (type(t2) not in (list, tuple)) or len(t2) not in (3, 4):
+            raise ValueError("bad row %i" % (i + 1))
+        if (type(t2[0]) is not int) or t2[0] < 1:
+            raise ValueError("bad hierarchy level in row %i" % (i + 1))
+        if t2[0] > t1[0] + 1:
+            raise ValueError("bad hierarchy level in row %i" % (i + 1))
+    # no formal errors in toc --------------------------------------------------
+
+    # --------------------------------------------------------------------------
+    # make a list of xref numbers, which we can use for our TOC entries
+    # --------------------------------------------------------------------------
+    old_xrefs = doc._delToC()  # del old outlines, get their xref numbers
+
+    # prepare table of xrefs for new bookmarks
+    old_xrefs = []
+    xref = [0] + old_xrefs
+    xref[0] = doc._getOLRootNumber()  # entry zero is outline root xref number
+    if toclen > len(old_xrefs):  # too few old xrefs?
+        for i in range((toclen - len(old_xrefs))):
+            xref.append(doc.get_new_xref())  # acquire new ones
+
+    lvltab = {0: 0}  # to store last entry per hierarchy level
+
+    # ------------------------------------------------------------------------------
+    # contains new outline objects as strings - first one is the outline root
+    # ------------------------------------------------------------------------------
+    olitems = [{"count": 0, "first": -1, "last": -1, "xref": xref[0]}]
+    # ------------------------------------------------------------------------------
+    # build olitems as a list of PDF-like connected dictionaries
+    # ------------------------------------------------------------------------------
+    for i in range(toclen):
+        o = toc[i]
+        lvl = o[0]  # level
+        title = pymupdf.get_pdf_str(o[1])  # title
+        pno = min(doc.page_count - 1, max(0, o[2] - 1))  # page number
+        page_xref = doc.page_xref(pno)
+        page_height = doc.page_cropbox(pno).height
+        top = pymupdf.Point(72, page_height - 36)
+        dest_dict = {"to": top, "kind": pymupdf.LINK_GOTO}  # fall back target
+        if o[2] < 0:
+            dest_dict["kind"] = pymupdf.LINK_NONE
+        if len(o) > 3:  # some target is specified
+            if type(o[3]) in (int, float):  # convert a number to a point
+                dest_dict["to"] = pymupdf.Point(72, page_height - o[3])
+            else:  # if something else, make sure we have a dict
+                # We make a copy of o[3] to avoid modifying our caller's data.
+                dest_dict = o[3].copy() if type(o[3]) is dict else dest_dict
+                if "to" not in dest_dict:  # target point not in dict?
+                    dest_dict["to"] = top  # put default in
+                else:  # transform target to PDF coordinates
+                    page = doc[pno]
+                    point = pymupdf.Point(dest_dict["to"])
+                    point.y = page.cropbox.height - point.y
+                    point = point * page.rotation_matrix
+                    dest_dict["to"] = (point.x, point.y)
+        d = {}
+        d["first"] = -1
+        d["count"] = 0
+        d["last"] = -1
+        d["prev"] = -1
+        d["next"] = -1
+        d["dest"] = getDestStr(page_xref, dest_dict)
+        d["top"] = dest_dict["to"]
+        d["title"] = title
+        d["parent"] = lvltab[lvl - 1]
+        d["xref"] = xref[i + 1]
+        d["color"] = dest_dict.get("color")
+        d["flags"] = dest_dict.get("italic", 0) + 2 * dest_dict.get("bold", 0)
+        lvltab[lvl] = i + 1
+        parent = olitems[lvltab[lvl - 1]]  # the parent entry
+
+        if (
+            dest_dict.get("collapse") or collapse and lvl > collapse
+        ):  # suppress expansion
+            parent["count"] -= 1  # make /Count negative
+        else:
+            parent["count"] += 1  # positive /Count
+
+        if parent["first"] == -1:
+            parent["first"] = i + 1
+            parent["last"] = i + 1
+        else:
+            d["prev"] = parent["last"]
+            prev = olitems[parent["last"]]
+            prev["next"] = i + 1
+            parent["last"] = i + 1
+        olitems.append(d)
+
+    # ------------------------------------------------------------------------------
+    # now create each outline item as a string and insert it in the PDF
+    # ------------------------------------------------------------------------------
+    for i, ol in enumerate(olitems):
+        txt = "<<"
+        if ol["count"] != 0:
+            txt += "/Count %i" % ol["count"]
+        try:
+            txt += ol["dest"]
+        except Exception:
+            # Verbose in PyMuPDF/tests.
+            if g_exceptions_verbose >= 2:   pymupdf.exception_info()
+            pass
+        try:
+            if ol["first"] > -1:
+                txt += "/First %i 0 R" % xref[ol["first"]]
+        except Exception:
+            if g_exceptions_verbose >= 2:   pymupdf.exception_info()
+            pass
+        try:
+            if ol["last"] > -1:
+                txt += "/Last %i 0 R" % xref[ol["last"]]
+        except Exception:
+            if g_exceptions_verbose >= 2:   pymupdf.exception_info()
+            pass
+        try:
+            if ol["next"] > -1:
+                txt += "/Next %i 0 R" % xref[ol["next"]]
+        except Exception:
+            # Verbose in PyMuPDF/tests.
+            if g_exceptions_verbose >= 2:   pymupdf.exception_info()
+            pass
+        try:
+            if ol["parent"] > -1:
+                txt += "/Parent %i 0 R" % xref[ol["parent"]]
+        except Exception:
+            # Verbose in PyMuPDF/tests.
+            if g_exceptions_verbose >= 2:   pymupdf.exception_info()
+            pass
+        try:
+            if ol["prev"] > -1:
+                txt += "/Prev %i 0 R" % xref[ol["prev"]]
+        except Exception:
+            # Verbose in PyMuPDF/tests.
+            if g_exceptions_verbose >= 2:   pymupdf.exception_info()
+            pass
+        try:
+            txt += "/Title" + ol["title"]
+        except Exception:
+            # Verbose in PyMuPDF/tests.
+            if g_exceptions_verbose >= 2:   pymupdf.exception_info()
+            pass
+
+        if ol.get("color") and len(ol["color"]) == 3:
+            txt += f"/C[ {_format_g(tuple(ol['color']))}]"
+        if ol.get("flags", 0) > 0:
+            txt += "/F %i" % ol["flags"]
+
+        if i == 0:  # special: this is the outline root
+            txt += "/Type/Outlines"  # so add the /Type entry
+        txt += ">>"
+        doc.update_object(xref[i], txt)  # insert the PDF object
+
+    doc.init_doc()
+    return toclen
+
+
+def do_widgets(
+    tar: pymupdf.Document,
+    src: pymupdf.Document,
+    graftmap,
+    from_page: int = -1,
+    to_page: int = -1,
+    start_at: int = -1,
+    join_duplicates=0,
+) -> None:
+    """Insert widgets of copied page range into target PDF.
+
+    Parameter values **must** equal those of method insert_pdf() which
+    must have been previously executed.
+    """
+    if not src.is_form_pdf:  # nothing to do: source PDF has no fields
+        return
+
+    def clean_kid_parents(acro_fields):
+        """ Make sure all kids have correct "Parent" pointers."""
+        for i in range(acro_fields.pdf_array_len()):
+            parent = acro_fields.pdf_array_get(i)
+            kids = parent.pdf_dict_get(pymupdf.PDF_NAME("Kids"))
+            for j in range(kids.pdf_array_len()):
+                kid = kids.pdf_array_get(j)
+                kid.pdf_dict_put(pymupdf.PDF_NAME("Parent"), parent)
+
+    def join_widgets(pdf, acro_fields, xref1, xref2, name):
+        """Called for each pair of widgets having the same name.
+
+        Args:
+            pdf: target MuPDF document
+            acro_fields: object Root/AcroForm/Fields
+            xref1, xref2: widget xrefs having same names
+            name: (str) the name
+
+        Result:
+            Defined or updated widget parent that points to both widgets.
+        """
+
+        def re_target(pdf, acro_fields, xref1, kids1, xref2, kids2):
+            """Merge widget in xref2 into "Kids" list of widget xref1.
+
+            Args:
+                xref1, kids1: target widget and its "Kids" array.
+                xref2, kids2: source wwidget and its "Kids" array (may be empty).
+            """
+            # make indirect objects from widgets
+            w1_ind = mupdf.pdf_new_indirect(pdf, xref1, 0)
+            w2_ind = mupdf.pdf_new_indirect(pdf, xref2, 0)
+            # find source widget in "Fields" array
+            idx = acro_fields.pdf_array_find(w2_ind)
+            acro_fields.pdf_array_delete(idx)
+
+            if not kids2.pdf_is_array():  # source widget has no kids
+                widget = mupdf.pdf_load_object(pdf, xref2)
+
+                # delete name from widget and insert target as parent
+                widget.pdf_dict_del(pymupdf.PDF_NAME("T"))
+                widget.pdf_dict_put(pymupdf.PDF_NAME("Parent"), w1_ind)
+
+                # put in target Kids
+                kids1.pdf_array_push(w2_ind)
+            else:  # copy source kids to target kids
+                for i in range(kids2.pdf_array_len()):
+                    kid = kids2.pdf_array_get(i)
+                    kid.pdf_dict_put(pymupdf.PDF_NAME("Parent"), w1_ind)
+                    kid_ind = mupdf.pdf_new_indirect(pdf, kid.pdf_to_num(), 0)
+                    kids1.pdf_array_push(kid_ind)
+
+        def new_target(pdf, acro_fields, xref1, w1, xref2, w2, name):
+            """Make new "Parent" for two widgets with same name.
+
+            Args:
+                xref1, w1: first widget
+                xref2, w2: second widget
+                name: field name
+
+            Result:
+                Both widgets have no "Kids". We create a new object with the
+                name and a "Kids" array containing the widgets.
+                Original widgets must be removed from AcroForm/Fields.
+            """
+            # make new "Parent" object
+            new = mupdf.pdf_new_dict(pdf, 5)
+            new.pdf_dict_put_text_string(pymupdf.PDF_NAME("T"), name)
+            kids = new.pdf_dict_put_array(pymupdf.PDF_NAME("Kids"), 2)
+            new_obj = mupdf.pdf_add_object(pdf, new)
+            new_obj_xref = new_obj.pdf_to_num()
+            new_ind = mupdf.pdf_new_indirect(pdf, new_obj_xref, 0)
+
+            # copy over some required source widget properties
+            ft = w1.pdf_dict_get(pymupdf.PDF_NAME("FT"))
+            w1.pdf_dict_del(pymupdf.PDF_NAME("FT"))
+            new_obj.pdf_dict_put(pymupdf.PDF_NAME("FT"), ft)
+
+            aa = w1.pdf_dict_get(pymupdf.PDF_NAME("AA"))
+            w1.pdf_dict_del(pymupdf.PDF_NAME("AA"))
+            new_obj.pdf_dict_put(pymupdf.PDF_NAME("AA"), aa)
+
+            # remove name field, insert "Parent" field in source widgets
+            w1.pdf_dict_del(pymupdf.PDF_NAME("T"))
+            w1.pdf_dict_put(pymupdf.PDF_NAME("Parent"), new_ind)
+            w2.pdf_dict_del(pymupdf.PDF_NAME("T"))
+            w2.pdf_dict_put(pymupdf.PDF_NAME("Parent"), new_ind)
+
+            # put source widgets in "kids" array
+            ind1 = mupdf.pdf_new_indirect(pdf, xref1, 0)
+            ind2 = mupdf.pdf_new_indirect(pdf, xref2, 0)
+            kids.pdf_array_push(ind1)
+            kids.pdf_array_push(ind2)
+
+            # remove source widgets from "AcroForm/Fields"
+            idx = acro_fields.pdf_array_find(ind1)
+            acro_fields.pdf_array_delete(idx)
+            idx = acro_fields.pdf_array_find(ind2)
+            acro_fields.pdf_array_delete(idx)
+
+            acro_fields.pdf_array_push(new_ind)
+
+        w1 = mupdf.pdf_load_object(pdf, xref1)
+        w2 = mupdf.pdf_load_object(pdf, xref2)
+        kids1 = w1.pdf_dict_get(pymupdf.PDF_NAME("Kids"))
+        kids2 = w2.pdf_dict_get(pymupdf.PDF_NAME("Kids"))
+
+        # check which widget has a suitable "Kids" array
+        if kids1.pdf_is_array():
+            re_target(pdf, acro_fields, xref1, kids1, xref2, kids2)  # pylint: disable=arguments-out-of-order
+        elif kids2.pdf_is_array():
+            re_target(pdf, acro_fields, xref2, kids2, xref1, kids1)  # pylint: disable=arguments-out-of-order
+        else:
+            new_target(pdf, acro_fields, xref1, w1, xref2, w2, name)  # pylint: disable=arguments-out-of-order
+
+    def get_kids(parent, kids_list):
+        """Return xref list of leaf kids for a parent.
+
+        Call with an empty list.
+        """
+        kids = mupdf.pdf_dict_get(parent, pymupdf.PDF_NAME("Kids"))
+        if not kids.pdf_is_array():
+            return kids_list
+        for i in range(kids.pdf_array_len()):
+            kid = kids.pdf_array_get(i)
+            if mupdf.pdf_is_dict(mupdf.pdf_dict_get(kid, pymupdf.PDF_NAME("Kids"))):
+                kids_list = get_kids(kid, kids_list)
+            else:
+                kids_list.append(kid.pdf_to_num())
+        return kids_list
+
+    def kids_xrefs(widget):
+        """Get the xref of top "Parent" and the list of leaf widgets."""
+        kids_list = []
+        parent = mupdf.pdf_dict_get(widget, pymupdf.PDF_NAME("Parent"))
+        parent_xref = parent.pdf_to_num()
+        if parent_xref == 0:
+            return parent_xref, kids_list
+        kids_list = get_kids(parent, kids_list)
+        return parent_xref, kids_list
+
+    def deduplicate_names(pdf, acro_fields, join_duplicates=False):
+        """Handle any widget name duplicates caused by the merge."""
+        names = {}  # key is a widget name, value a list of widgets having it.
+
+        # extract all names and widgets in "AcroForm/Fields"
+        for i in range(mupdf.pdf_array_len(acro_fields)):
+            wobject = mupdf.pdf_array_get(acro_fields, i)
+            xref = wobject.pdf_to_num()
+
+            # extract widget name and collect widget(s) using it
+            T = mupdf.pdf_dict_get_text_string(wobject, pymupdf.PDF_NAME("T"))
+            xrefs = names.get(T, [])
+            xrefs.append(xref)
+            names[T] = xrefs
+
+        for name, xrefs in names.items():
+            if len(xrefs) < 2:
+                continue
+            xref0, xref1 = xrefs[:2]  # only exactly 2 should occur!
+            if join_duplicates:  # combine fields with equal names
+                join_widgets(pdf, acro_fields, xref0, xref1, name)
+            else:  # make field names unique
+                newname = name + f" [{xref1}]"  # append this to the name
+                wobject = mupdf.pdf_load_object(pdf, xref1)
+                wobject.pdf_dict_put_text_string(pymupdf.PDF_NAME("T"), newname)
+
+        clean_kid_parents(acro_fields)
+
+    def get_acroform(doc):
+        """Retrieve the AcroForm dictionary form a PDF."""
+        pdf = mupdf.pdf_document_from_fz_document(doc)
+        # AcroForm (= central form field info)
+        return mupdf.pdf_dict_getp(mupdf.pdf_trailer(pdf), "Root/AcroForm")
+
+    tarpdf = mupdf.pdf_document_from_fz_document(tar)
+    srcpdf = mupdf.pdf_document_from_fz_document(src)
+
+    if tar.is_form_pdf:
+        # target is a Form PDF, so use it to include source fields
+        acro = get_acroform(tar)
+        # Important arrays in AcroForm
+        acro_fields = acro.pdf_dict_get(pymupdf.PDF_NAME("Fields"))
+        tar_co = acro.pdf_dict_get(pymupdf.PDF_NAME("CO"))
+        if not tar_co.pdf_is_array():
+            tar_co = acro.pdf_dict_put_array(pymupdf.PDF_NAME("CO"), 5)
+    else:
+        # target is no Form PDF, so copy over source AcroForm
+        acro = mupdf.pdf_deep_copy_obj(get_acroform(src))  # make a copy
+
+        # Clear "Fields" and "CO" arrays: will be populated by page fields.
+        # This is required to avoid copying unneeded objects.
+        acro.pdf_dict_del(pymupdf.PDF_NAME("Fields"))
+        acro.pdf_dict_put_array(pymupdf.PDF_NAME("Fields"), 5)
+        acro.pdf_dict_del(pymupdf.PDF_NAME("CO"))
+        acro.pdf_dict_put_array(pymupdf.PDF_NAME("CO"), 5)
+
+        # Enrich AcroForm for copying to target
+        acro_graft = mupdf.pdf_graft_mapped_object(graftmap, acro)
+
+        # Insert AcroForm into target PDF
+        acro_tar = mupdf.pdf_add_object(tarpdf, acro_graft)
+        acro_fields = acro_tar.pdf_dict_get(pymupdf.PDF_NAME("Fields"))
+        tar_co = acro_tar.pdf_dict_get(pymupdf.PDF_NAME("CO"))
+
+        # get its xref and insert it into target catalog
+        tar_xref = acro_tar.pdf_to_num()
+        acro_tar_ind = mupdf.pdf_new_indirect(tarpdf, tar_xref, 0)
+        root = mupdf.pdf_dict_get(mupdf.pdf_trailer(tarpdf), pymupdf.PDF_NAME("Root"))
+        root.pdf_dict_put(pymupdf.PDF_NAME("AcroForm"), acro_tar_ind)
+
+    if from_page <= to_page:
+        src_range = range(from_page, to_page + 1)
+    else:
+        src_range = range(from_page, to_page - 1, -1)
+
+    parents = {}  # information about widget parents
+
+    # remove "P" owning page reference from all widgets of all source pages
+    for i in src_range:
+        src_page = src[i]
+        for xref in [
+            xref
+            for xref, wtype, _ in src_page.annot_xrefs()
+            if wtype == pymupdf.PDF_ANNOT_WIDGET  # pylint: disable=no-member
+        ]:
+            w_obj = mupdf.pdf_load_object(srcpdf, xref)
+            w_obj.pdf_dict_del(pymupdf.PDF_NAME("P"))
+
+            # get the widget's parent structure
+            parent_xref, old_kids = kids_xrefs(w_obj)
+            if parent_xref:
+                parents[parent_xref] = {
+                    "new_xref": 0,
+                    "old_kids": old_kids,
+                    "new_kids": [],
+                }
+    # Copy over Parent widgets first - they are not page-dependent
+    for xref in parents.keys():  # pylint: disable=consider-using-dict-items
+        parent = mupdf.pdf_load_object(srcpdf, xref)
+        parent_graft = mupdf.pdf_graft_mapped_object(graftmap, parent)
+        parent_tar = mupdf.pdf_add_object(tarpdf, parent_graft)
+        kids_xrefs_new = get_kids(parent_tar, [])
+        parent_xref_new = parent_tar.pdf_to_num()
+        parent_ind = mupdf.pdf_new_indirect(tarpdf, parent_xref_new, 0)
+        acro_fields.pdf_array_push(parent_ind)
+        parents[xref]["new_xref"] = parent_xref_new
+        parents[xref]["new_kids"] = kids_xrefs_new
+
+    for i in range(len(src_range)):
+        # read first copied over page in target
+        tar_page = tar[start_at + i]
+
+        # read the original page in the source PDF
+        src_page = src[src_range[i]]
+
+        # now walk through source page widgets and copy over
+        w_xrefs = [  # widget xrefs of the source page
+            xref
+            for xref, wtype, _ in src_page.annot_xrefs()
+            if wtype == pymupdf.PDF_ANNOT_WIDGET  # pylint: disable=no-member
+        ]
+        if not w_xrefs:  # no widgets on this source page
+            continue
+
+        # convert to formal PDF page
+        tar_page_pdf = mupdf.pdf_page_from_fz_page(tar_page)
+
+        # extract annotations array
+        tar_annots = mupdf.pdf_dict_get(tar_page_pdf.obj(), pymupdf.PDF_NAME("Annots"))
+        if not mupdf.pdf_is_array(tar_annots):
+            tar_annots = mupdf.pdf_dict_put_array(
+                tar_page_pdf.obj(), pymupdf.PDF_NAME("Annots"), 5
+            )
+
+        for xref in w_xrefs:
+            w_obj = mupdf.pdf_load_object(srcpdf, xref)
+
+            # check if field takes part in inter-field validations
+            is_aac = mupdf.pdf_is_dict(mupdf.pdf_dict_getp(w_obj, "AA/C"))
+
+            # check if parent of widget already in target
+            parent_xref = mupdf.pdf_to_num(
+                w_obj.pdf_dict_get(pymupdf.PDF_NAME("Parent"))
+            )
+            if parent_xref == 0:  # parent not in target yet
+                w_obj_graft = mupdf.pdf_graft_mapped_object(graftmap, w_obj)
+                w_obj_tar = mupdf.pdf_add_object(tarpdf, w_obj_graft)
+                tar_xref = w_obj_tar.pdf_to_num()
+                w_obj_tar_ind = mupdf.pdf_new_indirect(tarpdf, tar_xref, 0)
+                mupdf.pdf_array_push(tar_annots, w_obj_tar_ind)
+                mupdf.pdf_array_push(acro_fields, w_obj_tar_ind)
+            else:
+                parent = parents[parent_xref]
+                idx = parent["old_kids"].index(xref)  # search for xref in parent
+                tar_xref = parent["new_kids"][idx]
+                w_obj_tar_ind = mupdf.pdf_new_indirect(tarpdf, tar_xref, 0)
+                mupdf.pdf_array_push(tar_annots, w_obj_tar_ind)
+
+            # Into "AcroForm/CO" if a computation field.
+            if is_aac:
+                mupdf.pdf_array_push(tar_co, w_obj_tar_ind)
+
+    deduplicate_names(tarpdf, acro_fields, join_duplicates=join_duplicates)
+
+def do_links(
+    doc1: pymupdf.Document,
+    doc2: pymupdf.Document,
+    from_page: int = -1,
+    to_page: int = -1,
+    start_at: int = -1,
+) -> None:
+    """Insert links contained in copied page range into destination PDF.
+
+    Parameter values **must** equal those of method insert_pdf(), which must
+    have been previously executed.
+    """
+    #pymupdf.log( 'utils.do_links()')
+    # --------------------------------------------------------------------------
+    # internal function to create the actual "/Annots" object string
+    # --------------------------------------------------------------------------
+    def cre_annot(lnk, xref_dst, pno_src, ctm):
+        """Create annotation object string for a passed-in link."""
+
+        r = lnk["from"] * ctm  # rect in PDF coordinates
+        rect = _format_g(tuple(r))
+        if lnk["kind"] == pymupdf.LINK_GOTO:
+            txt = pymupdf.annot_skel["goto1"]  # annot_goto
+            idx = pno_src.index(lnk["page"])
+            p = lnk["to"] * ctm  # target point in PDF coordinates
+            annot = txt(xref_dst[idx], p.x, p.y, lnk["zoom"], rect)
+
+        elif lnk["kind"] == pymupdf.LINK_GOTOR:
+            if lnk["page"] >= 0:
+                txt = pymupdf.annot_skel["gotor1"]  # annot_gotor
+                pnt = lnk.get("to", pymupdf.Point(0, 0))  # destination point
+                if type(pnt) is not pymupdf.Point:
+                    pnt = pymupdf.Point(0, 0)
+                annot = txt(
+                    lnk["page"],
+                    pnt.x,
+                    pnt.y,
+                    lnk["zoom"],
+                    lnk["file"],
+                    lnk["file"],
+                    rect,
+                )
+            else:
+                txt = pymupdf.annot_skel["gotor2"]  # annot_gotor_n
+                to = pymupdf.get_pdf_str(lnk["to"])
+                to = to[1:-1]
+                f = lnk["file"]
+                annot = txt(to, f, rect)
+
+        elif lnk["kind"] == pymupdf.LINK_LAUNCH:
+            txt = pymupdf.annot_skel["launch"]  # annot_launch
+            annot = txt(lnk["file"], lnk["file"], rect)
+
+        elif lnk["kind"] == pymupdf.LINK_URI:
+            txt = pymupdf.annot_skel["uri"]  # annot_uri
+            annot = txt(lnk["uri"], rect)
+
+        else:
+            annot = ""
+
+        return annot
+
+    # --------------------------------------------------------------------------
+
+    # validate & normalize parameters
+    if from_page < 0:
+        fp = 0
+    elif from_page >= doc2.page_count:
+        fp = doc2.page_count - 1
+    else:
+        fp = from_page
+
+    if to_page < 0 or to_page >= doc2.page_count:
+        tp = doc2.page_count - 1
+    else:
+        tp = to_page
+
+    if start_at < 0:
+        raise ValueError("'start_at' must be >= 0")
+    sa = start_at
+
+    incr = 1 if fp <= tp else -1  # page range could be reversed
+
+    # lists of source / destination page numbers
+    pno_src = list(range(fp, tp + incr, incr))
+    pno_dst = [sa + i for i in range(len(pno_src))]
+
+    # lists of source / destination page xrefs
+    xref_src = []
+    xref_dst = []
+    for i in range(len(pno_src)):
+        p_src = pno_src[i]
+        p_dst = pno_dst[i]
+        old_xref = doc2.page_xref(p_src)
+        new_xref = doc1.page_xref(p_dst)
+        xref_src.append(old_xref)
+        xref_dst.append(new_xref)
+
+    # create the links for each copied page in destination PDF
+    for i in range(len(xref_src)):
+        page_src = doc2[pno_src[i]]  # load source page
+        links = page_src.get_links()  # get all its links
+        #pymupdf.log( '{pno_src=}')
+        #pymupdf.log( '{type(page_src)=}')
+        #pymupdf.log( '{page_src=}')
+        #pymupdf.log( '{=i len(links)}')
+        if len(links) == 0:  # no links there
+            page_src = None
+            continue
+        ctm = ~page_src.transformation_matrix  # calc page transformation matrix
+        page_dst = doc1[pno_dst[i]]  # load destination page
+        link_tab = []  # store all link definitions here
+        for l in links:
+            if l["kind"] == pymupdf.LINK_GOTO and (l["page"] not in pno_src):
+                continue  # GOTO link target not in copied pages
+            annot_text = cre_annot(l, xref_dst, pno_src, ctm)
+            if annot_text:
+                link_tab.append(annot_text)
+        if link_tab != []:
+            page_dst._addAnnot_FromString( tuple(link_tab))
+    #pymupdf.log( 'utils.do_links() returning.')
+
+
+def getLinkText(page: pymupdf.Page, lnk: dict) -> str:
+    # --------------------------------------------------------------------------
+    # define skeletons for /Annots object texts
+    # --------------------------------------------------------------------------
+    ctm = page.transformation_matrix
+    ictm = ~ctm
+    r = lnk["from"]
+    rect = _format_g(tuple(r * ictm))
+
+    annot = ""
+    if lnk["kind"] == pymupdf.LINK_GOTO:
+        if lnk["page"] >= 0:
+            txt = pymupdf.annot_skel["goto1"]  # annot_goto
+            pno = lnk["page"]
+            xref = page.parent.page_xref(pno)
+            pnt = lnk.get("to", pymupdf.Point(0, 0))  # destination point
+            dest_page = page.parent[pno]
+            dest_ctm = dest_page.transformation_matrix
+            dest_ictm = ~dest_ctm
+            ipnt = pnt * dest_ictm
+            annot = txt(xref, ipnt.x, ipnt.y, lnk.get("zoom", 0), rect)
+        else:
+            txt = pymupdf.annot_skel["goto2"]  # annot_goto_n
+            annot = txt(pymupdf.get_pdf_str(lnk["to"]), rect)
+
+    elif lnk["kind"] == pymupdf.LINK_GOTOR:
+        if lnk["page"] >= 0:
+            txt = pymupdf.annot_skel["gotor1"]  # annot_gotor
+            pnt = lnk.get("to", pymupdf.Point(0, 0))  # destination point
+            if type(pnt) is not pymupdf.Point:
+                pnt = pymupdf.Point(0, 0)
+            annot = txt(
+                lnk["page"],
+                pnt.x,
+                pnt.y,
+                lnk.get("zoom", 0),
+                lnk["file"],
+                lnk["file"],
+                rect,
+            )
+        else:
+            txt = pymupdf.annot_skel["gotor2"]  # annot_gotor_n
+            annot = txt(pymupdf.get_pdf_str(lnk["to"]), lnk["file"], rect)
+
+    elif lnk["kind"] == pymupdf.LINK_LAUNCH:
+        txt = pymupdf.annot_skel["launch"]  # annot_launch
+        annot = txt(lnk["file"], lnk["file"], rect)
+
+    elif lnk["kind"] == pymupdf.LINK_URI:
+        txt = pymupdf.annot_skel["uri"]  # txt = annot_uri
+        annot = txt(lnk["uri"], rect)
+
+    elif lnk["kind"] == pymupdf.LINK_NAMED:
+        txt = pymupdf.annot_skel["named"]  # annot_named
+        lname = lnk.get("name")  # check presence of key
+        if lname is None:  # if missing, fall back to alternative
+            lname = lnk["nameddest"]
+        annot = txt(lname, rect)
+    if not annot:
+        return annot
+
+    # add a /NM PDF key to the object definition
+    link_names = dict(  # existing ids and their xref
+        [(x[0], x[2]) for x in page.annot_xrefs() if x[1] == pymupdf.PDF_ANNOT_LINK]   # pylint: disable=no-member
+    )
+
+    old_name = lnk.get("id", "")  # id value in the argument
+
+    if old_name and (lnk["xref"], old_name) in link_names.items():
+        name = old_name  # no new name if this is an update only
+    else:
+        i = 0
+        stem = pymupdf.TOOLS.set_annot_stem() + "-L%i"
+        while True:
+            name = stem % i
+            if name not in link_names.values():
+                break
+            i += 1
+    # add /NM key to object definition
+    annot = annot.replace("/Link", "/Link/NM(%s)" % name)
+    return annot
+
+
+def delete_widget(page: pymupdf.Page, widget: pymupdf.Widget) -> pymupdf.Widget:
+    """Delete widget from page and return the next one."""
+    pymupdf.CheckParent(page)
+    annot = getattr(widget, "_annot", None)
+    if annot is None:
+        raise ValueError("bad type: widget")
+    nextwidget = widget.next
+    page.delete_annot(annot)
+    widget._annot.parent = None
+    keylist = list(widget.__dict__.keys())
+    for key in keylist:
+        del widget.__dict__[key]
+    return nextwidget
+
+
+def update_link(page: pymupdf.Page, lnk: dict) -> None:
+    """Update a link on the current page."""
+    pymupdf.CheckParent(page)
+    annot = getLinkText(page, lnk)
+    if annot == "":
+        raise ValueError("link kind not supported")
+
+    page.parent.update_object(lnk["xref"], annot, page=page)
+
+
+def insert_link(page: pymupdf.Page, lnk: dict, mark: bool = True) -> None:
+    """Insert a new link for the current page."""
+    pymupdf.CheckParent(page)
+    annot = getLinkText(page, lnk)
+    if annot == "":
+        raise ValueError("link kind not supported")
+    page._addAnnot_FromString((annot,))
+
+
+def insert_textbox(
+    page: pymupdf.Page,
+    rect: rect_like,
+    buffer: typing.Union[str, list],
+    *,
+    fontname: str = "helv",
+    fontfile: OptStr = None,
+    set_simple: int = 0,
+    encoding: int = 0,
+    fontsize: float = 11,
+    lineheight: OptFloat = None,
+    color: OptSeq = None,
+    fill: OptSeq = None,
+    expandtabs: int = 1,
+    align: int = 0,
+    rotate: int = 0,
+    render_mode: int = 0,
+    miter_limit: float = 1,
+    border_width: float = 0.05,
+    morph: OptSeq = None,
+    overlay: bool = True,
+    stroke_opacity: float = 1,
+    fill_opacity: float = 1,
+    oc: int = 0,
+) -> float:
+    """Insert text into a given rectangle.
+
+    Notes:
+        Creates a Shape object, uses its same-named method and commits it.
+    Parameters:
+        rect: (rect-like) area to use for text.
+        buffer: text to be inserted
+        fontname: a Base-14 font, font name or '/name'
+        fontfile: name of a font file
+        fontsize: font size
+        lineheight: overwrite the font property
+        color: RGB color triple
+        expandtabs: handles tabulators with string function
+        align: left, center, right, justified
+        rotate: 0, 90, 180, or 270 degrees
+        morph: morph box with a matrix and a fixpoint
+        overlay: put text in foreground or background
+    Returns:
+        unused or deficit rectangle area (float)
+    """
+    img = page.new_shape()
+    rc = img.insert_textbox(
+        rect,
+        buffer,
+        fontsize=fontsize,
+        lineheight=lineheight,
+        fontname=fontname,
+        fontfile=fontfile,
+        set_simple=set_simple,
+        encoding=encoding,
+        color=color,
+        fill=fill,
+        expandtabs=expandtabs,
+        render_mode=render_mode,
+        miter_limit=miter_limit,
+        border_width=border_width,
+        align=align,
+        rotate=rotate,
+        morph=morph,
+        stroke_opacity=stroke_opacity,
+        fill_opacity=fill_opacity,
+        oc=oc,
+    )
+    if rc >= 0:
+        img.commit(overlay)
+    return rc
+
+
+def insert_text(
+    page: pymupdf.Page,
+    point: point_like,
+    text: typing.Union[str, list],
+    *,
+    fontsize: float = 11,
+    lineheight: OptFloat = None,
+    fontname: str = "helv",
+    fontfile: OptStr = None,
+    set_simple: int = 0,
+    encoding: int = 0,
+    color: OptSeq = None,
+    fill: OptSeq = None,
+    border_width: float = 0.05,
+    miter_limit: float = 1,
+    render_mode: int = 0,
+    rotate: int = 0,
+    morph: OptSeq = None,
+    overlay: bool = True,
+    stroke_opacity: float = 1,
+    fill_opacity: float = 1,
+    oc: int = 0,
+):
+
+    img = page.new_shape()
+    rc = img.insert_text(
+        point,
+        text,
+        fontsize=fontsize,
+        lineheight=lineheight,
+        fontname=fontname,
+        fontfile=fontfile,
+        set_simple=set_simple,
+        encoding=encoding,
+        color=color,
+        fill=fill,
+        border_width=border_width,
+        render_mode=render_mode,
+        miter_limit=miter_limit,
+        rotate=rotate,
+        morph=morph,
+        stroke_opacity=stroke_opacity,
+        fill_opacity=fill_opacity,
+        oc=oc,
+    )
+    if rc >= 0:
+        img.commit(overlay)
+    return rc
+
+
+def insert_htmlbox(
+    page,
+    rect,
+    text,
+    *,
+    css=None,
+    scale_low=0,
+    archive=None,
+    rotate=0,
+    oc=0,
+    opacity=1,
+    overlay=True,
+) -> float:
+    """Insert text with optional HTML tags and stylings into a rectangle.
+
+    Args:
+        rect: (rect-like) rectangle into which the text should be placed.
+        text: (str) text with optional HTML tags and stylings.
+        css: (str) CSS styling commands.
+        scale_low: (float) force-fit content by scaling it down. Must be in
+            range [0, 1]. If 1, no scaling will take place. If 0, arbitrary
+            down-scaling is acceptable. A value of 0.1 would mean that content
+            may be scaled down by at most 90%.
+        archive: Archive object pointing to locations of used fonts or images
+        rotate: (int) rotate the text in the box by a multiple of 90 degrees.
+        oc: (int) the xref of an OCG / OCMD (Optional Content).
+        opacity: (float) set opacity of inserted content.
+        overlay: (bool) put text on top of page content.
+    Returns:
+        A tuple of floats (spare_height, scale).
+        spare_height: -1 if content did not fit, else >= 0. It is the height of the
+               unused (still available) rectangle stripe. Positive only if
+               scale_min = 1 (no down scaling).
+        scale: downscaling factor, 0 < scale <= 1. Set to 0 if spare_height = -1 (no fit).
+    """
+
+    # normalize rotation angle
+    if not rotate % 90 == 0:
+        raise ValueError("bad rotation angle")
+    while rotate < 0:
+        rotate += 360
+    while rotate >= 360:
+        rotate -= 360
+
+    if not 0 <= scale_low <= 1:
+        raise ValueError("'scale_low' must be in [0, 1]")
+
+    if css is None:
+        css = ""
+
+    rect = pymupdf.Rect(rect)
+    if rotate in (90, 270):
+        temp_rect = pymupdf.Rect(0, 0, rect.height, rect.width)
+    else:
+        temp_rect = pymupdf.Rect(0, 0, rect.width, rect.height)
+
+    # use a small border by default
+    mycss = "body {margin:1px;}" + css  # append user CSS
+
+    # either make a story, or accept a given one
+    if isinstance(text, str):  # if a string, convert to a Story
+        story = pymupdf.Story(html=text, user_css=mycss, archive=archive)
+    elif isinstance(text, pymupdf.Story):
+        story = text
+    else:
+        raise ValueError("'text' must be a string or a Story")
+    # ----------------------------------------------------------------
+    # Find a scaling factor that lets our story fit in
+    # ----------------------------------------------------------------
+    scale_max = None if scale_low == 0 else 1 / scale_low
+
+    fit = story.fit_scale(temp_rect, scale_min=1, scale_max=scale_max)
+    if not fit.big_enough:  # there was no fit
+        return (-1, scale_low)
+
+    filled = fit.filled
+    scale = 1 / fit.parameter  # shrink factor
+
+    spare_height = fit.rect.y1 - filled[3]  # unused room at rectangle bottom
+    # Note: due to MuPDF's logic this may be negative even for successful fits.
+    if scale != 1 or spare_height < 0:  # if scaling occurred, set spare_height to 0
+        spare_height = 0
+
+    def rect_function(*args):
+        return fit.rect, fit.rect, pymupdf.Identity
+
+    # draw story on temp PDF page
+    doc = story.write_with_links(rect_function)
+
+    # Insert opacity if requested.
+    # For this, we prepend a command to the /Contents.
+    if 0 <= opacity < 1:
+        tpage = doc[0]  # load page
+        # generate /ExtGstate for the page
+        alp0 = tpage._set_opacity(CA=opacity, ca=opacity)
+        s = f"/{alp0} gs\n"  # generate graphic state command
+        pymupdf.TOOLS._insert_contents(tpage, s.encode(), 0)
+
+    # put result in target page
+    page.show_pdf_page(rect, doc, 0, rotate=rotate, oc=oc, overlay=overlay)
+
+    # -------------------------------------------------------------------------
+    # re-insert links in target rect (show_pdf_page cannot copy annotations)
+    # -------------------------------------------------------------------------
+    # scaled center point of fit.rect
+    mp1 = (fit.rect.tl + fit.rect.br) / 2 * scale
+
+    # center point of target rect
+    mp2 = (rect.tl + rect.br) / 2
+
+    # compute link positioning matrix:
+    # - move center of scaled-down fit.rect to (0,0)
+    # - rotate
+    # - move (0,0) to center of target rect
+    mat = (
+        pymupdf.Matrix(scale, 0, 0, scale, -mp1.x, -mp1.y)
+        * pymupdf.Matrix(-rotate)
+        * pymupdf.Matrix(1, 0, 0, 1, mp2.x, mp2.y)
+    )
+
+    # copy over links
+    for link in doc[0].get_links():
+        link["from"] *= mat
+        page.insert_link(link)
+
+    return spare_height, scale
+
+
+def new_page(
+    doc: pymupdf.Document,
+    pno: int = -1,
+    width: float = 595,
+    height: float = 842,
+) -> pymupdf.Page:
+    """Create and return a new page object.
+
+    Args:
+        pno: (int) insert before this page. Default: after last page.
+        width: (float) page width in points. Default: 595 (ISO A4 width).
+        height: (float) page height in points. Default 842 (ISO A4 height).
+    Returns:
+        A pymupdf.Page object.
+    """
+    doc._newPage(pno, width=width, height=height)
+    return doc[pno]
+
+
+def insert_page(
+    doc: pymupdf.Document,
+    pno: int,
+    text: typing.Union[str, list, None] = None,
+    fontsize: float = 11,
+    width: float = 595,
+    height: float = 842,
+    fontname: str = "helv",
+    fontfile: OptStr = None,
+    color: OptSeq = (0,),
+) -> int:
+    """Create a new PDF page and insert some text.
+
+    Notes:
+        Function combining pymupdf.Document.new_page() and pymupdf.Page.insert_text().
+        For parameter details see these methods.
+    """
+    page = doc.new_page(pno=pno, width=width, height=height)
+    if not bool(text):
+        return 0
+    rc = page.insert_text(
+        (50, 72),
+        text,
+        fontsize=fontsize,
+        fontname=fontname,
+        fontfile=fontfile,
+        color=color,
+    )
+    return rc
+
+
+def draw_line(
+    page: pymupdf.Page,
+    p1: point_like,
+    p2: point_like,
+    color: OptSeq = (0,),
+    dashes: OptStr = None,
+    width: float = 1,
+    lineCap: int = 0,
+    lineJoin: int = 0,
+    overlay: bool = True,
+    morph: OptSeq = None,
+    stroke_opacity: float = 1,
+    fill_opacity: float = 1,
+    oc=0,
+) -> pymupdf.Point:
+    """Draw a line from point p1 to point p2."""
+    img = page.new_shape()
+    p = img.draw_line(pymupdf.Point(p1), pymupdf.Point(p2))
+    img.finish(
+        color=color,
+        dashes=dashes,
+        width=width,
+        closePath=False,
+        lineCap=lineCap,
+        lineJoin=lineJoin,
+        morph=morph,
+        stroke_opacity=stroke_opacity,
+        fill_opacity=fill_opacity,
+        oc=oc,
+    )
+    img.commit(overlay)
+
+    return p
+
+
+def draw_squiggle(
+    page: pymupdf.Page,
+    p1: point_like,
+    p2: point_like,
+    breadth: float = 2,
+    color: OptSeq = (0,),
+    dashes: OptStr = None,
+    width: float = 1,
+    lineCap: int = 0,
+    lineJoin: int = 0,
+    overlay: bool = True,
+    morph: OptSeq = None,
+    stroke_opacity: float = 1,
+    fill_opacity: float = 1,
+    oc: int = 0,
+) -> pymupdf.Point:
+    """Draw a squiggly line from point p1 to point p2."""
+    img = page.new_shape()
+    p = img.draw_squiggle(pymupdf.Point(p1), pymupdf.Point(p2), breadth=breadth)
+    img.finish(
+        color=color,
+        dashes=dashes,
+        width=width,
+        closePath=False,
+        lineCap=lineCap,
+        lineJoin=lineJoin,
+        morph=morph,
+        stroke_opacity=stroke_opacity,
+        fill_opacity=fill_opacity,
+        oc=oc,
+    )
+    img.commit(overlay)
+
+    return p
+
+
+def draw_zigzag(
+    page: pymupdf.Page,
+    p1: point_like,
+    p2: point_like,
+    breadth: float = 2,
+    color: OptSeq = (0,),
+    dashes: OptStr = None,
+    width: float = 1,
+    lineCap: int = 0,
+    lineJoin: int = 0,
+    overlay: bool = True,
+    morph: OptSeq = None,
+    stroke_opacity: float = 1,
+    fill_opacity: float = 1,
+    oc: int = 0,
+) -> pymupdf.Point:
+    """Draw a zigzag line from point p1 to point p2."""
+    img = page.new_shape()
+    p = img.draw_zigzag(pymupdf.Point(p1), pymupdf.Point(p2), breadth=breadth)
+    img.finish(
+        color=color,
+        dashes=dashes,
+        width=width,
+        closePath=False,
+        lineCap=lineCap,
+        lineJoin=lineJoin,
+        morph=morph,
+        stroke_opacity=stroke_opacity,
+        fill_opacity=fill_opacity,
+        oc=oc,
+    )
+    img.commit(overlay)
+
+    return p
+
+
+def draw_rect(
+        page: pymupdf.Page,
+        rect: rect_like,
+        color: OptSeq = (0,),
+        fill: OptSeq = None,
+        dashes: OptStr = None,
+        width: float = 1,
+        lineCap: int = 0,
+        lineJoin: int = 0,
+        morph: OptSeq = None,
+        overlay: bool = True,
+        stroke_opacity: float = 1,
+        fill_opacity: float = 1,
+        oc: int = 0,
+        radius=None,
+        ) -> pymupdf.Point:
+    '''
+    Draw a rectangle. See Shape class method for details.
+    '''
+    img = page.new_shape()
+    Q = img.draw_rect(pymupdf.Rect(rect), radius=radius)
+    img.finish(
+        color=color,
+        fill=fill,
+        dashes=dashes,
+        width=width,
+        lineCap=lineCap,
+        lineJoin=lineJoin,
+        morph=morph,
+        stroke_opacity=stroke_opacity,
+        fill_opacity=fill_opacity,
+        oc=oc,
+    )
+    img.commit(overlay)
+
+    return Q
+
+
+def draw_quad(
+    page: pymupdf.Page,
+    quad: quad_like,
+    color: OptSeq = (0,),
+    fill: OptSeq = None,
+    dashes: OptStr = None,
+    width: float = 1,
+    lineCap: int = 0,
+    lineJoin: int = 0,
+    morph: OptSeq = None,
+    overlay: bool = True,
+    stroke_opacity: float = 1,
+    fill_opacity: float = 1,
+    oc: int = 0,
+) -> pymupdf.Point:
+    """Draw a quadrilateral."""
+    img = page.new_shape()
+    Q = img.draw_quad(pymupdf.Quad(quad))
+    img.finish(
+        color=color,
+        fill=fill,
+        dashes=dashes,
+        width=width,
+        lineCap=lineCap,
+        lineJoin=lineJoin,
+        morph=morph,
+        stroke_opacity=stroke_opacity,
+        fill_opacity=fill_opacity,
+        oc=oc,
+    )
+    img.commit(overlay)
+
+    return Q
+
+
+def draw_polyline(
+    page: pymupdf.Page,
+    points: list,
+    color: OptSeq = (0,),
+    fill: OptSeq = None,
+    dashes: OptStr = None,
+    width: float = 1,
+    morph: OptSeq = None,
+    lineCap: int = 0,
+    lineJoin: int = 0,
+    overlay: bool = True,
+    closePath: bool = False,
+    stroke_opacity: float = 1,
+    fill_opacity: float = 1,
+    oc: int = 0,
+) -> pymupdf.Point:
+    """Draw multiple connected line segments."""
+    img = page.new_shape()
+    Q = img.draw_polyline(points)
+    img.finish(
+        color=color,
+        fill=fill,
+        dashes=dashes,
+        width=width,
+        lineCap=lineCap,
+        lineJoin=lineJoin,
+        morph=morph,
+        closePath=closePath,
+        stroke_opacity=stroke_opacity,
+        fill_opacity=fill_opacity,
+        oc=oc,
+    )
+    img.commit(overlay)
+
+    return Q
+
+
+def draw_circle(
+    page: pymupdf.Page,
+    center: point_like,
+    radius: float,
+    color: OptSeq = (0,),
+    fill: OptSeq = None,
+    morph: OptSeq = None,
+    dashes: OptStr = None,
+    width: float = 1,
+    lineCap: int = 0,
+    lineJoin: int = 0,
+    overlay: bool = True,
+    stroke_opacity: float = 1,
+    fill_opacity: float = 1,
+    oc: int = 0,
+) -> pymupdf.Point:
+    """Draw a circle given its center and radius."""
+    img = page.new_shape()
+    Q = img.draw_circle(pymupdf.Point(center), radius)
+    img.finish(
+        color=color,
+        fill=fill,
+        dashes=dashes,
+        width=width,
+        lineCap=lineCap,
+        lineJoin=lineJoin,
+        morph=morph,
+        stroke_opacity=stroke_opacity,
+        fill_opacity=fill_opacity,
+        oc=oc,
+    )
+    img.commit(overlay)
+    return Q
+
+
+def draw_oval(
+    page: pymupdf.Page,
+    rect: typing.Union[rect_like, quad_like],
+    color: OptSeq = (0,),
+    fill: OptSeq = None,
+    dashes: OptStr = None,
+    morph: OptSeq = None,
+    width: float = 1,
+    lineCap: int = 0,
+    lineJoin: int = 0,
+    overlay: bool = True,
+    stroke_opacity: float = 1,
+    fill_opacity: float = 1,
+    oc: int = 0,
+) -> pymupdf.Point:
+    """Draw an oval given its containing rectangle or quad."""
+    img = page.new_shape()
+    Q = img.draw_oval(rect)
+    img.finish(
+        color=color,
+        fill=fill,
+        dashes=dashes,
+        width=width,
+        lineCap=lineCap,
+        lineJoin=lineJoin,
+        morph=morph,
+        stroke_opacity=stroke_opacity,
+        fill_opacity=fill_opacity,
+        oc=oc,
+    )
+    img.commit(overlay)
+
+    return Q
+
+
+def draw_curve(
+    page: pymupdf.Page,
+    p1: point_like,
+    p2: point_like,
+    p3: point_like,
+    color: OptSeq = (0,),
+    fill: OptSeq = None,
+    dashes: OptStr = None,
+    width: float = 1,
+    morph: OptSeq = None,
+    closePath: bool = False,
+    lineCap: int = 0,
+    lineJoin: int = 0,
+    overlay: bool = True,
+    stroke_opacity: float = 1,
+    fill_opacity: float = 1,
+    oc: int = 0,
+) -> pymupdf.Point:
+    """Draw a special Bezier curve from p1 to p3, generating control points on lines p1 to p2 and p2 to p3."""
+    img = page.new_shape()
+    Q = img.draw_curve(pymupdf.Point(p1), pymupdf.Point(p2), pymupdf.Point(p3))
+    img.finish(
+        color=color,
+        fill=fill,
+        dashes=dashes,
+        width=width,
+        lineCap=lineCap,
+        lineJoin=lineJoin,
+        morph=morph,
+        closePath=closePath,
+        stroke_opacity=stroke_opacity,
+        fill_opacity=fill_opacity,
+        oc=oc,
+    )
+    img.commit(overlay)
+
+    return Q
+
+
+def draw_bezier(
+    page: pymupdf.Page,
+    p1: point_like,
+    p2: point_like,
+    p3: point_like,
+    p4: point_like,
+    color: OptSeq = (0,),
+    fill: OptSeq = None,
+    dashes: OptStr = None,
+    width: float = 1,
+    morph: OptStr = None,
+    closePath: bool = False,
+    lineCap: int = 0,
+    lineJoin: int = 0,
+    overlay: bool = True,
+    stroke_opacity: float = 1,
+    fill_opacity: float = 1,
+    oc: int = 0,
+) -> pymupdf.Point:
+    """Draw a general cubic Bezier curve from p1 to p4 using control points p2 and p3."""
+    img = page.new_shape()
+    Q = img.draw_bezier(pymupdf.Point(p1), pymupdf.Point(p2), pymupdf.Point(p3), pymupdf.Point(p4))
+    img.finish(
+        color=color,
+        fill=fill,
+        dashes=dashes,
+        width=width,
+        lineCap=lineCap,
+        lineJoin=lineJoin,
+        morph=morph,
+        closePath=closePath,
+        stroke_opacity=stroke_opacity,
+        fill_opacity=fill_opacity,
+        oc=oc,
+    )
+    img.commit(overlay)
+
+    return Q
+
+
+def draw_sector(
+    page: pymupdf.Page,
+    center: point_like,
+    point: point_like,
+    beta: float,
+    color: OptSeq = (0,),
+    fill: OptSeq = None,
+    dashes: OptStr = None,
+    fullSector: bool = True,
+    morph: OptSeq = None,
+    width: float = 1,
+    closePath: bool = False,
+    lineCap: int = 0,
+    lineJoin: int = 0,
+    overlay: bool = True,
+    stroke_opacity: float = 1,
+    fill_opacity: float = 1,
+    oc: int = 0,
+) -> pymupdf.Point:
+    """Draw a circle sector given circle center, one arc end point and the angle of the arc.
+
+    Parameters:
+        center -- center of circle
+        point -- arc end point
+        beta -- angle of arc (degrees)
+        fullSector -- connect arc ends with center
+    """
+    img = page.new_shape()
+    Q = img.draw_sector(pymupdf.Point(center), pymupdf.Point(point), beta, fullSector=fullSector)
+    img.finish(
+        color=color,
+        fill=fill,
+        dashes=dashes,
+        width=width,
+        lineCap=lineCap,
+        lineJoin=lineJoin,
+        morph=morph,
+        closePath=closePath,
+        stroke_opacity=stroke_opacity,
+        fill_opacity=fill_opacity,
+        oc=oc,
+    )
+    img.commit(overlay)
+
+    return Q
+
+
+# ----------------------------------------------------------------------
+# Name:        wx.lib.colourdb.py
+# Purpose:     Adds a bunch of colour names and RGB values to the
+#              colour database so they can be found by name
+#
+# Author:      Robin Dunn
+#
+# Created:     13-March-2001
+# Copyright:   (c) 2001-2017 by Total Control Software
+# Licence:     wxWindows license
+# Tags:        phoenix-port, unittest, documented
+# ----------------------------------------------------------------------
+
+
+def getColorList() -> list:
+    """
+    Returns a list of upper-case colour names.
+    :rtype: list of strings
+    """
+    return [name for name, r, g, b in pymupdf.colors_wx_list()]
+
+
+def getColorInfoList() -> list:
+    """
+    Returns list of (name, red, gree, blue) tuples, where:
+        name: upper-case color name.
+        read, green, blue: integers in range 0..255.
+    :rtype: list of tuples
+    """
+    return pymupdf.colors_wx_list()
+
+
+def getColor(name: str) -> tuple:
+    """Retrieve RGB color in PDF format by name.
+
+    Returns:
+        a triple of floats in range 0 to 1. In case of name-not-found, "white" is returned.
+    """
+    return pymupdf.colors_pdf_dict().get(name.lower(), (1, 1, 1))
+
+
+def getColorHSV(name: str) -> tuple:
+    """Retrieve the hue, saturation, value triple of a color name.
+
+    Returns:
+        a triple (degree, percent, percent). If not found (-1, -1, -1) is returned.
+    """
+    try:
+        x = getColorInfoList()[getColorList().index(name.upper())]
+    except Exception:
+        if g_exceptions_verbose:    pymupdf.exception_info()
+        return (-1, -1, -1)
+
+    r = x[1] / 255.0
+    g = x[2] / 255.0
+    b = x[3] / 255.0
+    cmax = max(r, g, b)
+    V = round(cmax * 100, 1)
+    cmin = min(r, g, b)
+    delta = cmax - cmin
+    if delta == 0:
+        hue = 0
+    elif cmax == r:
+        hue = 60.0 * (((g - b) / delta) % 6)
+    elif cmax == g:
+        hue = 60.0 * (((b - r) / delta) + 2)
+    else:
+        hue = 60.0 * (((r - g) / delta) + 4)
+
+    H = int(round(hue))
+
+    if cmax == 0:
+        sat = 0
+    else:
+        sat = delta / cmax
+    S = int(round(sat * 100))
+
+    return (H, S, V)
+
+
+def _get_font_properties(doc: pymupdf.Document, xref: int) -> tuple:
+    fontname, ext, stype, buffer = doc.extract_font(xref)
+    asc = 0.8
+    dsc = -0.2
+    if ext == "":
+        return fontname, ext, stype, asc, dsc
+
+    if buffer:
+        try:
+            font = pymupdf.Font(fontbuffer=buffer)
+            asc = font.ascender
+            dsc = font.descender
+            bbox = font.bbox
+            if asc - dsc < 1:
+                if bbox.y0 < dsc:
+                    dsc = bbox.y0
+                asc = 1 - dsc
+        except Exception:
+            pymupdf.exception_info()
+            asc *= 1.2
+            dsc *= 1.2
+        return fontname, ext, stype, asc, dsc
+    if ext != "n/a":
+        try:
+            font = pymupdf.Font(fontname)
+            asc = font.ascender
+            dsc = font.descender
+        except Exception:
+            pymupdf.exception_info()
+            asc *= 1.2
+            dsc *= 1.2
+    else:
+        asc *= 1.2
+        dsc *= 1.2
+    return fontname, ext, stype, asc, dsc
+
+
+def get_char_widths(
+    doc: pymupdf.Document, xref: int, limit: int = 256, idx: int = 0, fontdict: OptDict = None
+) -> list:
+    """Get list of glyph information of a font.
+
+    Notes:
+        Must be provided by its XREF number. If we already dealt with the
+        font, it will be recorded in doc.FontInfos. Otherwise we insert an
+        entry there.
+        Finally we return the glyphs for the font. This is a list of
+        (glyph, width) where glyph is an integer controlling the char
+        appearance, and width is a float controlling the char's spacing:
+        width * fontsize is the actual space.
+        For 'simple' fonts, glyph == ord(char) will usually be true.
+        Exceptions are 'Symbol' and 'ZapfDingbats'. We are providing data for these directly here.
+    """
+    fontinfo = pymupdf.CheckFontInfo(doc, xref)
+    if fontinfo is None:  # not recorded yet: create it
+        if fontdict is None:
+            name, ext, stype, asc, dsc = _get_font_properties(doc, xref)
+            fontdict = {
+                "name": name,
+                "type": stype,
+                "ext": ext,
+                "ascender": asc,
+                "descender": dsc,
+            }
+        else:
+            name = fontdict["name"]
+            ext = fontdict["ext"]
+            stype = fontdict["type"]
+            ordering = fontdict["ordering"]
+            simple = fontdict["simple"]
+
+        if ext == "":
+            raise ValueError("xref is not a font")
+
+        # check for 'simple' fonts
+        if stype in ("Type1", "MMType1", "TrueType"):
+            simple = True
+        else:
+            simple = False
+
+        # check for CJK fonts
+        if name in ("Fangti", "Ming"):
+            ordering = 0
+        elif name in ("Heiti", "Song"):
+            ordering = 1
+        elif name in ("Gothic", "Mincho"):
+            ordering = 2
+        elif name in ("Dotum", "Batang"):
+            ordering = 3
+        else:
+            ordering = -1
+
+        fontdict["simple"] = simple
+
+        if name == "ZapfDingbats":
+            glyphs = pymupdf.zapf_glyphs
+        elif name == "Symbol":
+            glyphs = pymupdf.symbol_glyphs
+        else:
+            glyphs = None
+
+        fontdict["glyphs"] = glyphs
+        fontdict["ordering"] = ordering
+        fontinfo = [xref, fontdict]
+        doc.FontInfos.append(fontinfo)
+    else:
+        fontdict = fontinfo[1]
+        glyphs = fontdict["glyphs"]
+        simple = fontdict["simple"]
+        ordering = fontdict["ordering"]
+
+    if glyphs is None:
+        oldlimit = 0
+    else:
+        oldlimit = len(glyphs)
+
+    mylimit = max(256, limit)
+
+    if mylimit <= oldlimit:
+        return glyphs
+
+    if ordering < 0:  # not a CJK font
+        glyphs = doc._get_char_widths(
+            xref, fontdict["name"], fontdict["ext"], fontdict["ordering"], mylimit, idx
+        )
+    else:  # CJK fonts use char codes and width = 1
+        glyphs = None
+
+    fontdict["glyphs"] = glyphs
+    fontinfo[1] = fontdict
+    pymupdf.UpdateFontInfo(doc, fontinfo)
+
+    return glyphs
+
+
+class Shape:
+    """Create a new shape."""
+
+    @staticmethod
+    def horizontal_angle(C, P):
+        """Return the angle to the horizontal for the connection from C to P.
+        This uses the arcus sine function and resolves its inherent ambiguity by
+        looking up in which quadrant vector S = P - C is located.
+        """
+        S = pymupdf.Point(P - C).unit  # unit vector 'C' -> 'P'
+        alfa = math.asin(abs(S.y))  # absolute angle from horizontal
+        if S.x < 0:  # make arcsin result unique
+            if S.y <= 0:  # bottom-left
+                alfa = -(math.pi - alfa)
+            else:  # top-left
+                alfa = math.pi - alfa
+        else:
+            if S.y >= 0:  # top-right
+                pass
+            else:  # bottom-right
+                alfa = -alfa
+        return alfa
+
+    def __init__(self, page: pymupdf.Page):
+        pymupdf.CheckParent(page)
+        self.page = page
+        self.doc = page.parent
+        if not self.doc.is_pdf:
+            raise ValueError("is no PDF")
+        self.height = page.mediabox_size.y
+        self.width = page.mediabox_size.x
+        self.x = page.cropbox_position.x
+        self.y = page.cropbox_position.y
+
+        self.pctm = page.transformation_matrix  # page transf. matrix
+        self.ipctm = ~self.pctm  # inverted transf. matrix
+
+        self.draw_cont = ""
+        self.text_cont = ""
+        self.totalcont = ""
+        self.last_point = None
+        self.rect = None
+
+    def updateRect(self, x):
+        if self.rect is None:
+            if len(x) == 2:
+                self.rect = pymupdf.Rect(x, x)
+            else:
+                self.rect = pymupdf.Rect(x)
+
+        else:
+            if len(x) == 2:
+                x = pymupdf.Point(x)
+                self.rect.x0 = min(self.rect.x0, x.x)
+                self.rect.y0 = min(self.rect.y0, x.y)
+                self.rect.x1 = max(self.rect.x1, x.x)
+                self.rect.y1 = max(self.rect.y1, x.y)
+            else:
+                x = pymupdf.Rect(x)
+                self.rect.x0 = min(self.rect.x0, x.x0)
+                self.rect.y0 = min(self.rect.y0, x.y0)
+                self.rect.x1 = max(self.rect.x1, x.x1)
+                self.rect.y1 = max(self.rect.y1, x.y1)
+
+    def draw_line(self, p1: point_like, p2: point_like) -> pymupdf.Point:
+        """Draw a line between two points."""
+        p1 = pymupdf.Point(p1)
+        p2 = pymupdf.Point(p2)
+        if not (self.last_point == p1):
+            self.draw_cont += _format_g(pymupdf.JM_TUPLE(p1 * self.ipctm)) + " m\n"
+            self.last_point = p1
+            self.updateRect(p1)
+
+        self.draw_cont += _format_g(pymupdf.JM_TUPLE(p2 * self.ipctm)) + " l\n"
+        self.updateRect(p2)
+        self.last_point = p2
+        return self.last_point
+
+    def draw_polyline(self, points: list) -> pymupdf.Point:
+        """Draw several connected line segments."""
+        for i, p in enumerate(points):
+            if i == 0:
+                if not (self.last_point == pymupdf.Point(p)):
+                    self.draw_cont += _format_g(pymupdf.JM_TUPLE(pymupdf.Point(p) * self.ipctm)) + " m\n"
+                    self.last_point = pymupdf.Point(p)
+            else:
+                self.draw_cont += _format_g(pymupdf.JM_TUPLE(pymupdf.Point(p) * self.ipctm)) + " l\n"
+            self.updateRect(p)
+
+        self.last_point = pymupdf.Point(points[-1])
+        return self.last_point
+
+    def draw_bezier(
+        self,
+        p1: point_like,
+        p2: point_like,
+        p3: point_like,
+        p4: point_like,
+    ) -> pymupdf.Point:
+        """Draw a standard cubic Bezier curve."""
+        p1 = pymupdf.Point(p1)
+        p2 = pymupdf.Point(p2)
+        p3 = pymupdf.Point(p3)
+        p4 = pymupdf.Point(p4)
+        if not (self.last_point == p1):
+            self.draw_cont += _format_g(pymupdf.JM_TUPLE(p1 * self.ipctm)) + " m\n"
+        args = pymupdf.JM_TUPLE(list(p2 * self.ipctm) + list(p3 * self.ipctm) + list(p4 * self.ipctm))
+        self.draw_cont += _format_g(args) + " c\n"
+        self.updateRect(p1)
+        self.updateRect(p2)
+        self.updateRect(p3)
+        self.updateRect(p4)
+        self.last_point = p4
+        return self.last_point
+
+    def draw_oval(self, tetra: typing.Union[quad_like, rect_like]) -> pymupdf.Point:
+        """Draw an ellipse inside a tetrapod."""
+        if len(tetra) != 4:
+            raise ValueError("invalid arg length")
+        if hasattr(tetra[0], "__float__"):
+            q = pymupdf.Rect(tetra).quad
+        else:
+            q = pymupdf.Quad(tetra)
+
+        mt = q.ul + (q.ur - q.ul) * 0.5
+        mr = q.ur + (q.lr - q.ur) * 0.5
+        mb = q.ll + (q.lr - q.ll) * 0.5
+        ml = q.ul + (q.ll - q.ul) * 0.5
+        if not (self.last_point == ml):
+            self.draw_cont += _format_g(pymupdf.JM_TUPLE(ml * self.ipctm)) + " m\n"
+            self.last_point = ml
+        self.draw_curve(ml, q.ll, mb)
+        self.draw_curve(mb, q.lr, mr)
+        self.draw_curve(mr, q.ur, mt)
+        self.draw_curve(mt, q.ul, ml)
+        self.updateRect(q.rect)
+        self.last_point = ml
+        return self.last_point
+
+    def draw_circle(self, center: point_like, radius: float) -> pymupdf.Point:
+        """Draw a circle given its center and radius."""
+        if not radius > pymupdf.EPSILON:
+            raise ValueError("radius must be positive")
+        center = pymupdf.Point(center)
+        p1 = center - (radius, 0)
+        return self.draw_sector(center, p1, 360, fullSector=False)
+
+    def draw_curve(
+        self,
+        p1: point_like,
+        p2: point_like,
+        p3: point_like,
+    ) -> pymupdf.Point:
+        """Draw a curve between points using one control point."""
+        kappa = 0.55228474983
+        p1 = pymupdf.Point(p1)
+        p2 = pymupdf.Point(p2)
+        p3 = pymupdf.Point(p3)
+        k1 = p1 + (p2 - p1) * kappa
+        k2 = p3 + (p2 - p3) * kappa
+        return self.draw_bezier(p1, k1, k2, p3)
+
+    def draw_sector(
+        self,
+        center: point_like,
+        point: point_like,
+        beta: float,
+        fullSector: bool = True,
+    ) -> pymupdf.Point:
+        """Draw a circle sector."""
+        center = pymupdf.Point(center)
+        point = pymupdf.Point(point)
+        l3 = lambda a, b: _format_g((a, b)) + " m\n"
+        l4 = lambda a, b, c, d, e, f: _format_g((a, b, c, d, e, f)) + " c\n"
+        l5 = lambda a, b: _format_g((a, b)) + " l\n"
+        betar = math.radians(-beta)
+        w360 = math.radians(math.copysign(360, betar)) * (-1)
+        w90 = math.radians(math.copysign(90, betar))
+        w45 = w90 / 2
+        while abs(betar) > 2 * math.pi:
+            betar += w360  # bring angle below 360 degrees
+        if not (self.last_point == point):
+            self.draw_cont += l3(*pymupdf.JM_TUPLE(point * self.ipctm))
+            self.last_point = point
+        Q = pymupdf.Point(0, 0)  # just make sure it exists
+        C = center
+        P = point
+        S = P - C  # vector 'center' -> 'point'
+        rad = abs(S)  # circle radius
+
+        if not rad > pymupdf.EPSILON:
+            raise ValueError("radius must be positive")
+
+        alfa = self.horizontal_angle(center, point)
+        while abs(betar) > abs(w90):  # draw 90 degree arcs
+            q1 = C.x + math.cos(alfa + w90) * rad
+            q2 = C.y + math.sin(alfa + w90) * rad
+            Q = pymupdf.Point(q1, q2)  # the arc's end point
+            r1 = C.x + math.cos(alfa + w45) * rad / math.cos(w45)
+            r2 = C.y + math.sin(alfa + w45) * rad / math.cos(w45)
+            R = pymupdf.Point(r1, r2)  # crossing point of tangents
+            kappah = (1 - math.cos(w45)) * 4 / 3 / abs(R - Q)
+            kappa = kappah * abs(P - Q)
+            cp1 = P + (R - P) * kappa  # control point 1
+            cp2 = Q + (R - Q) * kappa  # control point 2
+            self.draw_cont += l4(*pymupdf.JM_TUPLE(
+                list(cp1 * self.ipctm) + list(cp2 * self.ipctm) + list(Q * self.ipctm)
+            ))
+
+            betar -= w90  # reduce param angle by 90 deg
+            alfa += w90  # advance start angle by 90 deg
+            P = Q  # advance to arc end point
+        # draw (remaining) arc
+        if abs(betar) > 1e-3:  # significant degrees left?
+            beta2 = betar / 2
+            q1 = C.x + math.cos(alfa + betar) * rad
+            q2 = C.y + math.sin(alfa + betar) * rad
+            Q = pymupdf.Point(q1, q2)  # the arc's end point
+            r1 = C.x + math.cos(alfa + beta2) * rad / math.cos(beta2)
+            r2 = C.y + math.sin(alfa + beta2) * rad / math.cos(beta2)
+            R = pymupdf.Point(r1, r2)  # crossing point of tangents
+            # kappa height is 4/3 of segment height
+            kappah = (1 - math.cos(beta2)) * 4 / 3 / abs(R - Q)  # kappa height
+            kappa = kappah * abs(P - Q) / (1 - math.cos(betar))
+            cp1 = P + (R - P) * kappa  # control point 1
+            cp2 = Q + (R - Q) * kappa  # control point 2
+            self.draw_cont += l4(*pymupdf.JM_TUPLE(
+                list(cp1 * self.ipctm) + list(cp2 * self.ipctm) + list(Q * self.ipctm)
+            ))
+        if fullSector:
+            self.draw_cont += l3(*pymupdf.JM_TUPLE(point * self.ipctm))
+            self.draw_cont += l5(*pymupdf.JM_TUPLE(center * self.ipctm))
+            self.draw_cont += l5(*pymupdf.JM_TUPLE(Q * self.ipctm))
+        self.last_point = Q
+        return self.last_point
+
+    def draw_rect(self, rect: rect_like, *, radius=None) -> pymupdf.Point:
+        """Draw a rectangle.
+
+        Args:
+            radius: if not None, the rectangle will have rounded corners.
+                This is the radius of the curvature, given as percentage of
+                the rectangle width or height. Valid are values 0 < v <= 0.5.
+                For a sequence of two values, the corners will have different
+                radii. Otherwise, the percentage will be computed from the
+                shorter side. A value of (0.5, 0.5) will draw an ellipse.
+        """
+        r = pymupdf.Rect(rect)
+        if radius is None:  # standard rectangle
+            self.draw_cont += _format_g(pymupdf.JM_TUPLE(
+                list(r.bl * self.ipctm) + [r.width, r.height]
+            )) + " re\n"
+            self.updateRect(r)
+            self.last_point = r.tl
+            return self.last_point
+        # rounded corners requested. This requires 1 or 2 values, each
+        # with 0 < value <= 0.5
+        if hasattr(radius, "__float__"):
+            if radius <= 0 or radius > 0.5:
+                raise ValueError(f"bad radius value {radius}.")
+            d = min(r.width, r.height) * radius
+            px = (d, 0)
+            py = (0, d)
+        elif hasattr(radius, "__len__") and len(radius) == 2:
+            rx, ry = radius
+            px = (rx * r.width, 0)
+            py = (0, ry * r.height)
+            if min(rx, ry) <= 0 or max(rx, ry) > 0.5:
+                raise ValueError(f"bad radius value {radius}.")
+        else:
+            raise ValueError(f"bad radius value {radius}.")
+
+        lp = self.draw_line(r.tl + py, r.bl - py)
+        lp = self.draw_curve(lp, r.bl, r.bl + px)
+
+        lp = self.draw_line(lp, r.br - px)
+        lp = self.draw_curve(lp, r.br, r.br - py)
+
+        lp = self.draw_line(lp, r.tr + py)
+        lp = self.draw_curve(lp, r.tr, r.tr - px)
+
+        lp = self.draw_line(lp, r.tl + px)
+        self.last_point = self.draw_curve(lp, r.tl, r.tl + py)
+
+        self.updateRect(r)
+        return self.last_point
+
+    def draw_quad(self, quad: quad_like) -> pymupdf.Point:
+        """Draw a Quad."""
+        q = pymupdf.Quad(quad)
+        return self.draw_polyline([q.ul, q.ll, q.lr, q.ur, q.ul])
+
+    def draw_zigzag(
+        self,
+        p1: point_like,
+        p2: point_like,
+        breadth: float = 2,
+    ) -> pymupdf.Point:
+        """Draw a zig-zagged line from p1 to p2."""
+        p1 = pymupdf.Point(p1)
+        p2 = pymupdf.Point(p2)
+        S = p2 - p1  # vector start - end
+        rad = abs(S)  # distance of points
+        cnt = 4 * int(round(rad / (4 * breadth), 0))  # always take full phases
+        if cnt < 4:
+            raise ValueError("points too close")
+        mb = rad / cnt  # revised breadth
+        matrix = pymupdf.Matrix(pymupdf.util_hor_matrix(p1, p2))  # normalize line to x-axis
+        i_mat = ~matrix  # get original position
+        points = []  # stores edges
+        for i in range(1, cnt):
+            if i % 4 == 1:  # point "above" connection
+                p = pymupdf.Point(i, -1) * mb
+            elif i % 4 == 3:  # point "below" connection
+                p = pymupdf.Point(i, 1) * mb
+            else:  # ignore others
+                continue
+            points.append(p * i_mat)
+        self.draw_polyline([p1] + points + [p2])  # add start and end points
+        return p2
+
+    def draw_squiggle(
+        self,
+        p1: point_like,
+        p2: point_like,
+        breadth=2,
+    ) -> pymupdf.Point:
+        """Draw a squiggly line from p1 to p2."""
+        p1 = pymupdf.Point(p1)
+        p2 = pymupdf.Point(p2)
+        S = p2 - p1  # vector start - end
+        rad = abs(S)  # distance of points
+        cnt = 4 * int(round(rad / (4 * breadth), 0))  # always take full phases
+        if cnt < 4:
+            raise ValueError("points too close")
+        mb = rad / cnt  # revised breadth
+        matrix = pymupdf.Matrix(pymupdf.util_hor_matrix(p1, p2))  # normalize line to x-axis
+        i_mat = ~matrix  # get original position
+        k = 2.4142135623765633  # y of draw_curve helper point
+
+        points = []  # stores edges
+        for i in range(1, cnt):
+            if i % 4 == 1:  # point "above" connection
+                p = pymupdf.Point(i, -k) * mb
+            elif i % 4 == 3:  # point "below" connection
+                p = pymupdf.Point(i, k) * mb
+            else:  # else on connection line
+                p = pymupdf.Point(i, 0) * mb
+            points.append(p * i_mat)
+
+        points = [p1] + points + [p2]
+        cnt = len(points)
+        i = 0
+        while i + 2 < cnt:
+            self.draw_curve(points[i], points[i + 1], points[i + 2])
+            i += 2
+        return p2
+
+    # ==============================================================================
+    # Shape.insert_text
+    # ==============================================================================
+    def insert_text(
+        self,
+        point: point_like,
+        buffer: typing.Union[str, list],
+        *,
+        fontsize: float = 11,
+        lineheight: OptFloat = None,
+        fontname: str = "helv",
+        fontfile: OptStr = None,
+        set_simple: bool = 0,
+        encoding: int = 0,
+        color: OptSeq = None,
+        fill: OptSeq = None,
+        render_mode: int = 0,
+        border_width: float = 0.05,
+        miter_limit: float = 1,
+        rotate: int = 0,
+        morph: OptSeq = None,
+        stroke_opacity: float = 1,
+        fill_opacity: float = 1,
+        oc: int = 0,
+    ) -> int:
+
+        # ensure 'text' is a list of strings, worth dealing with
+        if not bool(buffer):
+            return 0
+
+        if type(buffer) not in (list, tuple):
+            text = buffer.splitlines()
+        else:
+            text = buffer
+
+        if not len(text) > 0:
+            return 0
+
+        point = pymupdf.Point(point)
+        try:
+            maxcode = max([ord(c) for c in " ".join(text)])
+        except Exception:
+            pymupdf.exception_info()
+            return 0
+
+        # ensure valid 'fontname'
+        fname = fontname
+        if fname.startswith("/"):
+            fname = fname[1:]
+
+        xref = self.page.insert_font(
+            fontname=fname, fontfile=fontfile, encoding=encoding, set_simple=set_simple
+        )
+        fontinfo = pymupdf.CheckFontInfo(self.doc, xref)
+
+        fontdict = fontinfo[1]
+        ordering = fontdict["ordering"]
+        simple = fontdict["simple"]
+        bfname = fontdict["name"]
+        ascender = fontdict["ascender"]
+        descender = fontdict["descender"]
+        if lineheight:
+            lheight = fontsize * lineheight
+        elif ascender - descender <= 1:
+            lheight = fontsize * 1.2
+        else:
+            lheight = fontsize * (ascender - descender)
+
+        if maxcode > 255:
+            glyphs = self.doc.get_char_widths(xref, maxcode + 1)
+        else:
+            glyphs = fontdict["glyphs"]
+
+        tab = []
+        for t in text:
+            if simple and bfname not in ("Symbol", "ZapfDingbats"):
+                g = None
+            else:
+                g = glyphs
+            tab.append(pymupdf.getTJstr(t, g, simple, ordering))
+        text = tab
+
+        color_str = pymupdf.ColorCode(color, "c")
+        fill_str = pymupdf.ColorCode(fill, "f")
+        if not fill and render_mode == 0:  # ensure fill color when 0 Tr
+            fill = color
+            fill_str = pymupdf.ColorCode(color, "f")
+
+        morphing = pymupdf.CheckMorph(morph)
+        rot = rotate
+        if rot % 90 != 0:
+            raise ValueError("bad rotate value")
+
+        while rot < 0:
+            rot += 360
+        rot = rot % 360  # text rotate = 0, 90, 270, 180
+
+        templ1 = lambda a, b, c, d, e, f, g: f"\nq\n{a}{b}BT\n{c}1 0 0 1 {_format_g((d, e))} Tm\n/{f} {_format_g(g)} Tf "
+        templ2 = lambda a: f"TJ\n0 -{_format_g(a)} TD\n"
+        cmp90 = "0 1 -1 0 0 0 cm\n"  # rotates 90 deg counter-clockwise
+        cmm90 = "0 -1 1 0 0 0 cm\n"  # rotates 90 deg clockwise
+        cm180 = "-1 0 0 -1 0 0 cm\n"  # rotates by 180 deg.
+        height = self.height
+        width = self.width
+
+        # setting up for standard rotation directions
+        # case rotate = 0
+        if morphing:
+            m1 = pymupdf.Matrix(1, 0, 0, 1, morph[0].x + self.x, height - morph[0].y - self.y)
+            mat = ~m1 * morph[1] * m1
+            cm = _format_g(pymupdf.JM_TUPLE(mat)) + " cm\n"
+        else:
+            cm = ""
+        top = height - point.y - self.y  # start of 1st char
+        left = point.x + self.x  # start of 1. char
+        space = top  # space available
+        #headroom = point.y + self.y  # distance to page border
+        if rot == 90:
+            left = height - point.y - self.y
+            top = -point.x - self.x
+            cm += cmp90
+            space = width - abs(top)
+            #headroom = point.x + self.x
+
+        elif rot == 270:
+            left = -height + point.y + self.y
+            top = point.x + self.x
+            cm += cmm90
+            space = abs(top)
+            #headroom = width - point.x - self.x
+
+        elif rot == 180:
+            left = -point.x - self.x
+            top = -height + point.y + self.y
+            cm += cm180
+            space = abs(point.y + self.y)
+            #headroom = height - point.y - self.y
+
+        optcont = self.page._get_optional_content(oc)
+        if optcont is not None:
+            bdc = "/OC /%s BDC\n" % optcont
+            emc = "EMC\n"
+        else:
+            bdc = emc = ""
+
+        alpha = self.page._set_opacity(CA=stroke_opacity, ca=fill_opacity)
+        if alpha is None:
+            alpha = ""
+        else:
+            alpha = "/%s gs\n" % alpha
+        nres = templ1(bdc, alpha, cm, left, top, fname, fontsize)
+
+        if render_mode > 0:
+            nres += "%i Tr " % render_mode
+            nres += _format_g(border_width * fontsize) + " w "
+            if miter_limit is not None:
+                nres += _format_g(miter_limit) + " M "
+        if color is not None:
+            nres += color_str
+        if fill is not None:
+            nres += fill_str
+
+        # =========================================================================
+        #   start text insertion
+        # =========================================================================
+        nres += text[0]
+        nlines = 1  # set output line counter
+        if len(text) > 1:
+            nres += templ2(lheight)  # line 1
+        else:
+            nres += 'TJ'
+        for i in range(1, len(text)):
+            if space < lheight:
+                break  # no space left on page
+            if i > 1:
+                nres += "\nT* "
+            nres += text[i] + 'TJ'
+            space -= lheight
+            nlines += 1
+
+        nres += "\nET\n%sQ\n" % emc
+
+        # =========================================================================
+        #   end of text insertion
+        # =========================================================================
+        # update the /Contents object
+        self.text_cont += nres
+        return nlines
+
+    # ==============================================================================
+    # Shape.insert_textbox
+    # ==============================================================================
+    def insert_textbox(
+        self,
+        rect: rect_like,
+        buffer: typing.Union[str, list],
+        *,
+        fontname: OptStr = "helv",
+        fontfile: OptStr = None,
+        fontsize: float = 11,
+        lineheight: OptFloat = None,
+        set_simple: bool = 0,
+        encoding: int = 0,
+        color: OptSeq = None,
+        fill: OptSeq = None,
+        expandtabs: int = 1,
+        border_width: float = 0.05,
+        miter_limit: float = 1,
+        align: int = 0,
+        render_mode: int = 0,
+        rotate: int = 0,
+        morph: OptSeq = None,
+        stroke_opacity: float = 1,
+        fill_opacity: float = 1,
+        oc: int = 0,
+    ) -> float:
+        """Insert text into a given rectangle.
+
+        Args:
+            rect -- the textbox to fill
+            buffer -- text to be inserted
+            fontname -- a Base-14 font, font name or '/name'
+            fontfile -- name of a font file
+            fontsize -- font size
+            lineheight -- overwrite the font property
+            color -- RGB stroke color triple
+            fill -- RGB fill color triple
+            render_mode -- text rendering control
+            border_width -- thickness of glyph borders as percentage of fontsize
+            expandtabs -- handles tabulators with string function
+            align -- left, center, right, justified
+            rotate -- 0, 90, 180, or 270 degrees
+            morph -- morph box with a matrix and a fixpoint
+        Returns:
+            unused or deficit rectangle area (float)
+        """
+        rect = pymupdf.Rect(rect)
+        if rect.is_empty or rect.is_infinite:
+            raise ValueError("text box must be finite and not empty")
+
+        color_str = pymupdf.ColorCode(color, "c")
+        fill_str = pymupdf.ColorCode(fill, "f")
+        if fill is None and render_mode == 0:  # ensure fill color for 0 Tr
+            fill = color
+            fill_str = pymupdf.ColorCode(color, "f")
+
+        optcont = self.page._get_optional_content(oc)
+        if optcont is not None:
+            bdc = "/OC /%s BDC\n" % optcont
+            emc = "EMC\n"
+        else:
+            bdc = emc = ""
+
+        # determine opacity / transparency
+        alpha = self.page._set_opacity(CA=stroke_opacity, ca=fill_opacity)
+        if alpha is None:
+            alpha = ""
+        else:
+            alpha = "/%s gs\n" % alpha
+
+        if rotate % 90 != 0:
+            raise ValueError("rotate must be multiple of 90")
+
+        rot = rotate
+        while rot < 0:
+            rot += 360
+        rot = rot % 360
+
+        # is buffer worth of dealing with?
+        if not bool(buffer):
+            return rect.height if rot in (0, 180) else rect.width
+
+        cmp90 = "0 1 -1 0 0 0 cm\n"  # rotates counter-clockwise
+        cmm90 = "0 -1 1 0 0 0 cm\n"  # rotates clockwise
+        cm180 = "-1 0 0 -1 0 0 cm\n"  # rotates by 180 deg.
+        height = self.height
+
+        fname = fontname
+        if fname.startswith("/"):
+            fname = fname[1:]
+
+        xref = self.page.insert_font(
+            fontname=fname, fontfile=fontfile, encoding=encoding, set_simple=set_simple
+        )
+        fontinfo = pymupdf.CheckFontInfo(self.doc, xref)
+
+        fontdict = fontinfo[1]
+        ordering = fontdict["ordering"]
+        simple = fontdict["simple"]
+        glyphs = fontdict["glyphs"]
+        bfname = fontdict["name"]
+        ascender = fontdict["ascender"]
+        descender = fontdict["descender"]
+
+        if lineheight:
+            lheight_factor = lineheight
+        elif ascender - descender <= 1:
+            lheight_factor = 1.2
+        else:
+            lheight_factor = ascender - descender
+        lheight = fontsize * lheight_factor
+
+        # create a list from buffer, split into its lines
+        if type(buffer) in (list, tuple):
+            t0 = "\n".join(buffer)
+        else:
+            t0 = buffer
+
+        maxcode = max([ord(c) for c in t0])
+        # replace invalid char codes for simple fonts
+        if simple and maxcode > 255:
+            t0 = "".join([c if ord(c) < 256 else "?" for c in t0])
+
+        t0 = t0.splitlines()
+
+        glyphs = self.doc.get_char_widths(xref, maxcode + 1)
+        if simple and bfname not in ("Symbol", "ZapfDingbats"):
+            tj_glyphs = None
+        else:
+            tj_glyphs = glyphs
+
+        # ----------------------------------------------------------------------
+        # calculate pixel length of a string
+        # ----------------------------------------------------------------------
+        def pixlen(x):
+            """Calculate pixel length of x."""
+            if ordering < 0:
+                return sum([glyphs[ord(c)][1] for c in x]) * fontsize
+            else:
+                return len(x) * fontsize
+
+        # ---------------------------------------------------------------------
+
+        if ordering < 0:
+            blen = glyphs[32][1] * fontsize  # pixel size of space character
+        else:
+            blen = fontsize
+
+        text = ""  # output buffer
+
+        if pymupdf.CheckMorph(morph):
+            m1 = pymupdf.Matrix(
+                1, 0, 0, 1, morph[0].x + self.x, self.height - morph[0].y - self.y
+            )
+            mat = ~m1 * morph[1] * m1
+            cm = _format_g(pymupdf.JM_TUPLE(mat)) + " cm\n"
+        else:
+            cm = ""
+
+        # ---------------------------------------------------------------------
+        # adjust for text orientation / rotation
+        # ---------------------------------------------------------------------
+        progr = 1  # direction of line progress
+        c_pnt = pymupdf.Point(0, fontsize * ascender)  # used for line progress
+        if rot == 0:  # normal orientation
+            point = rect.tl + c_pnt  # line 1 is 'lheight' below top
+            maxwidth = rect.width  # pixels available in one line
+            maxheight = rect.height  # available text height
+
+        elif rot == 90:  # rotate counter clockwise
+            c_pnt = pymupdf.Point(fontsize * ascender, 0)  # progress in x-direction
+            point = rect.bl + c_pnt  # line 1 'lheight' away from left
+            maxwidth = rect.height  # pixels available in one line
+            maxheight = rect.width  # available text height
+            cm += cmp90
+
+        elif rot == 180:  # text upside down
+            # progress upwards in y direction
+            c_pnt = -pymupdf.Point(0, fontsize * ascender)
+            point = rect.br + c_pnt  # line 1 'lheight' above bottom
+            maxwidth = rect.width  # pixels available in one line
+            progr = -1  # subtract lheight for next line
+            maxheight =rect.height  # available text height
+            cm += cm180
+
+        else:  # rotate clockwise (270 or -90)
+            # progress from right to left
+            c_pnt = -pymupdf.Point(fontsize * ascender, 0)
+            point = rect.tr + c_pnt  # line 1 'lheight' left of right
+            maxwidth = rect.height  # pixels available in one line
+            progr = -1  # subtract lheight for next line
+            maxheight = rect.width  # available text height
+            cm += cmm90
+
+        # =====================================================================
+        # line loop
+        # =====================================================================
+        just_tab = []  # 'justify' indicators per line
+
+        for i, line in enumerate(t0):
+            line_t = line.expandtabs(expandtabs).split(" ")  # split into words
+            num_words = len(line_t)
+            lbuff = ""  # init line buffer
+            rest = maxwidth  # available line pixels
+            # =================================================================
+            # word loop
+            # =================================================================
+            for j in range(num_words):
+                word = line_t[j]
+                pl_w = pixlen(word)  # pixel len of word
+                if rest >= pl_w:  # does it fit on the line?
+                    lbuff += word + " "  # yes, append word
+                    rest -= pl_w + blen  # update available line space
+                    continue  # next word
+
+                # word doesn't fit - output line (if not empty)
+                if lbuff:
+                    lbuff = lbuff.rstrip() + "\n"  # line full, append line break
+                    text += lbuff  # append to total text
+                    just_tab.append(True)  # can align-justify
+
+                lbuff = ""  # re-init line buffer
+                rest = maxwidth  # re-init avail. space
+
+                if pl_w <= maxwidth:  # word shorter than 1 line?
+                    lbuff = word + " "  # start the line with it
+                    rest = maxwidth - pl_w - blen  # update free space
+                    continue
+
+                # long word: split across multiple lines - char by char ...
+                if len(just_tab) > 0:
+                    just_tab[-1] = False  # cannot align-justify
+                for c in word:
+                    if pixlen(lbuff) <= maxwidth - pixlen(c):
+                        lbuff += c
+                    else:  # line full
+                        lbuff += "\n"  # close line
+                        text += lbuff  # append to text
+                        just_tab.append(False)  # cannot align-justify
+                        lbuff = c  # start new line with this char
+
+                lbuff += " "  # finish long word
+                rest = maxwidth - pixlen(lbuff)  # long word stored
+
+            if lbuff:  # unprocessed line content?
+                text += lbuff.rstrip()  # append to text
+                just_tab.append(False)  # cannot align-justify
+
+            if i < len(t0) - 1:  # not the last line?
+                text += "\n"  # insert line break
+
+        # compute used part of the textbox
+        if text.endswith("\n"):
+            text = text[:-1]
+        lb_count = text.count("\n") + 1  # number of lines written
+
+        # text height = line count * line height plus one descender value
+        text_height = lheight * lb_count - descender * fontsize
+
+        more = text_height - maxheight  # difference to height limit
+        if more > pymupdf.EPSILON:  # landed too much outside rect
+            return (-1) * more  # return deficit, don't output
+
+        more = abs(more)
+        if more < pymupdf.EPSILON:
+            more = 0  # don't bother with epsilons
+        nres = "\nq\n%s%sBT\n" % (bdc, alpha) + cm  # initialize output buffer
+        templ = lambda a, b, c, d: f"1 0 0 1 {_format_g((a, b))} Tm /{c} {_format_g(d)} Tf "
+        # center, right, justify: output each line with its own specifics
+        text_t = text.splitlines()  # split text in lines again
+        just_tab[-1] = False  # never justify last line
+        for i, t in enumerate(text_t):
+            spacing = 0
+            pl = maxwidth - pixlen(t)  # length of empty line part
+            pnt = point + c_pnt * (i * lheight_factor)  # text start of line
+            if align == 1:  # center: right shift by half width
+                if rot in (0, 180):
+                    pnt = pnt + pymupdf.Point(pl / 2, 0) * progr
+                else:
+                    pnt = pnt - pymupdf.Point(0, pl / 2) * progr
+            elif align == 2:  # right: right shift by full width
+                if rot in (0, 180):
+                    pnt = pnt + pymupdf.Point(pl, 0) * progr
+                else:
+                    pnt = pnt - pymupdf.Point(0, pl) * progr
+            elif align == 3:  # justify
+                spaces = t.count(" ")  # number of spaces in line
+                if spaces > 0 and just_tab[i]:  # if any, and we may justify
+                    spacing = pl / spaces  # make every space this much larger
+                else:
+                    spacing = 0  # keep normal space length
+            top = height - pnt.y - self.y
+            left = pnt.x + self.x
+            if rot == 90:
+                left = height - pnt.y - self.y
+                top = -pnt.x - self.x
+            elif rot == 270:
+                left = -height + pnt.y + self.y
+                top = pnt.x + self.x
+            elif rot == 180:
+                left = -pnt.x - self.x
+                top = -height + pnt.y + self.y
+
+            nres += templ(left, top, fname, fontsize)
+
+            if render_mode > 0:
+                nres += "%i Tr " % render_mode
+                nres += _format_g(border_width * fontsize) + " w "
+                if miter_limit is not None:
+                    nres += _format_g(miter_limit) + " M "
+
+            if align == 3:
+                nres += _format_g(spacing) + " Tw "
+
+            if color is not None:
+                nres += color_str
+            if fill is not None:
+                nres += fill_str
+            nres += "%sTJ\n" % pymupdf.getTJstr(t, tj_glyphs, simple, ordering)
+
+        nres += "ET\n%sQ\n" % emc
+
+        self.text_cont += nres
+        self.updateRect(rect)
+        return more
+
+    def finish(
+        self,
+        width: float = 1,
+        color: OptSeq = (0,),
+        fill: OptSeq = None,
+        lineCap: int = 0,
+        lineJoin: int = 0,
+        dashes: OptStr = None,
+        even_odd: bool = False,
+        morph: OptSeq = None,
+        closePath: bool = True,
+        fill_opacity: float = 1,
+        stroke_opacity: float = 1,
+        oc: int = 0,
+    ) -> None:
+        """Finish the current drawing segment.
+
+        Notes:
+            Apply colors, opacity, dashes, line style and width, or
+            morphing. Also whether to close the path
+            by connecting last to first point.
+        """
+        if self.draw_cont == "":  # treat empty contents as no-op
+            return
+
+        if width == 0:  # border color makes no sense then
+            color = None
+        elif color is None:  # vice versa
+            width = 0
+        # if color == None and fill == None:
+        #     raise ValueError("at least one of 'color' or 'fill' must be given")
+        color_str = pymupdf.ColorCode(color, "c")  # ensure proper color string
+        fill_str = pymupdf.ColorCode(fill, "f")  # ensure proper fill string
+
+        optcont = self.page._get_optional_content(oc)
+        if optcont is not None:
+            self.draw_cont = "/OC /%s BDC\n" % optcont + self.draw_cont
+            emc = "EMC\n"
+        else:
+            emc = ""
+
+        alpha = self.page._set_opacity(CA=stroke_opacity, ca=fill_opacity)
+        if alpha is not None:
+            self.draw_cont = "/%s gs\n" % alpha + self.draw_cont
+
+        if width != 1 and width != 0:
+            self.draw_cont += _format_g(width) + " w\n"
+
+        if lineCap != 0:
+            self.draw_cont = "%i J\n" % lineCap + self.draw_cont
+        if lineJoin != 0:
+            self.draw_cont = "%i j\n" % lineJoin + self.draw_cont
+
+        if dashes not in (None, "", "[] 0"):
+            self.draw_cont = "%s d\n" % dashes + self.draw_cont
+
+        if closePath:
+            self.draw_cont += "h\n"
+            self.last_point = None
+
+        if color is not None:
+            self.draw_cont += color_str
+
+        if fill is not None:
+            self.draw_cont += fill_str
+            if color is not None:
+                if not even_odd:
+                    self.draw_cont += "B\n"
+                else:
+                    self.draw_cont += "B*\n"
+            else:
+                if not even_odd:
+                    self.draw_cont += "f\n"
+                else:
+                    self.draw_cont += "f*\n"
+        else:
+            self.draw_cont += "S\n"
+
+        self.draw_cont += emc
+        if pymupdf.CheckMorph(morph):
+            m1 = pymupdf.Matrix(
+                1, 0, 0, 1, morph[0].x + self.x, self.height - morph[0].y - self.y
+            )
+            mat = ~m1 * morph[1] * m1
+            self.draw_cont = _format_g(pymupdf.JM_TUPLE(mat)) + " cm\n" + self.draw_cont
+
+        self.totalcont += "\nq\n" + self.draw_cont + "Q\n"
+        self.draw_cont = ""
+        self.last_point = None
+        return
+
+    def commit(self, overlay: bool = True) -> None:
+        """Update the page's /Contents object with Shape data.
+
+        The argument controls whether data appear in foreground (default)
+        or background.
+        """
+        pymupdf.CheckParent(self.page)  # doc may have died meanwhile
+        self.totalcont += self.text_cont
+        self.totalcont = self.totalcont.encode()
+
+        if self.totalcont:
+            if overlay:
+                self.page.wrap_contents()  # ensure a balanced graphics state
+            # make /Contents object with dummy stream
+            xref = pymupdf.TOOLS._insert_contents(self.page, b" ", overlay)
+            # update it with potential compression
+            self.doc.update_stream(xref, self.totalcont)
+
+        self.last_point = None  # clean up ...
+        self.rect = None  #
+        self.draw_cont = ""  # for potential ...
+        self.text_cont = ""  # ...
+        self.totalcont = ""  # re-use
+
+
+def apply_redactions(
+    page: pymupdf.Page, images: int = 2, graphics: int = 1, text: int = 0
+) -> bool:
+    """Apply the redaction annotations of the page.
+
+    Args:
+        page: the PDF page.
+        images:
+              0 - ignore images
+              1 - remove all overlapping images
+              2 - blank out overlapping image parts
+              3 - remove image unless invisible
+        graphics:
+              0 - ignore graphics
+              1 - remove graphics if contained in rectangle
+              2 - remove all overlapping graphics
+        text:
+              0 - remove text
+              1 - ignore text
+    """
+
+    def center_rect(annot_rect, new_text, font, fsize):
+        """Calculate minimal sub-rectangle for the overlay text.
+
+        Notes:
+            Because 'insert_textbox' supports no vertical text centering,
+            we calculate an approximate number of lines here and return a
+            sub-rect with smaller height, which should still be sufficient.
+        Args:
+            annot_rect: the annotation rectangle
+            new_text: the text to insert.
+            font: the fontname. Must be one of the CJK or Base-14 set, else
+                the rectangle is returned unchanged.
+            fsize: the fontsize
+        Returns:
+            A rectangle to use instead of the annot rectangle.
+        """
+        if not new_text or annot_rect.width <= pymupdf.EPSILON:
+            return annot_rect
+        try:
+            text_width = pymupdf.get_text_length(new_text, font, fsize)
+        except (ValueError, mupdf.FzErrorBase):  # unsupported font
+            if g_exceptions_verbose:
+                pymupdf.exception_info()
+            return annot_rect
+        line_height = fsize * 1.2
+        limit = annot_rect.width
+        h = math.ceil(text_width / limit) * line_height  # estimate rect height
+        if h >= annot_rect.height:
+            return annot_rect
+        r = annot_rect
+        y = (annot_rect.tl.y + annot_rect.bl.y - h) * 0.5
+        r.y0 = y
+        return r
+
+    pymupdf.CheckParent(page)
+    doc = page.parent
+    if doc.is_encrypted or doc.is_closed:
+        raise ValueError("document closed or encrypted")
+    if not doc.is_pdf:
+        raise ValueError("is no PDF")
+
+    redact_annots = []  # storage of annot values
+    for annot in page.annots(
+        types=(pymupdf.PDF_ANNOT_REDACT,)  # pylint: disable=no-member
+    ):
+        # loop redactions
+        redact_annots.append(annot._get_redact_values())  # save annot values
+
+    if redact_annots == []:  # any redactions on this page?
+        return False  # no redactions
+
+    rc = page._apply_redactions(text, images, graphics)  # call MuPDF
+    if not rc:  # should not happen really
+        raise ValueError("Error applying redactions.")
+
+    # now write replacement text in old redact rectangles
+    shape = page.new_shape()
+    for redact in redact_annots:
+        annot_rect = redact["rect"]
+        fill = redact["fill"]
+        if fill:
+            shape.draw_rect(annot_rect)  # colorize the rect background
+            shape.finish(fill=fill, color=fill)
+        if "text" in redact.keys():  # if we also have text
+            new_text = redact["text"]
+            align = redact.get("align", 0)
+            fname = redact["fontname"]
+            fsize = redact["fontsize"]
+            color = redact["text_color"]
+            # try finding vertical centered sub-rect
+            trect = center_rect(annot_rect, new_text, fname, fsize)
+
+            rc = -1
+            while rc < 0 and fsize >= 4:  # while not enough room
+                # (re-) try insertion
+                rc = shape.insert_textbox(
+                    trect,
+                    new_text,
+                    fontname=fname,
+                    fontsize=fsize,
+                    color=color,
+                    align=align,
+                )
+                fsize -= 0.5  # reduce font if unsuccessful
+    shape.commit()  # append new contents object
+    return True
+
+
+# ------------------------------------------------------------------------------
+# Remove potentially sensitive data from a PDF. Similar to the Adobe
+# Acrobat 'sanitize' function
+# ------------------------------------------------------------------------------
+def scrub(
+    doc: pymupdf.Document,
+    attached_files: bool = True,
+    clean_pages: bool = True,
+    embedded_files: bool = True,
+    hidden_text: bool = True,
+    javascript: bool = True,
+    metadata: bool = True,
+    redactions: bool = True,
+    redact_images: int = 0,
+    remove_links: bool = True,
+    reset_fields: bool = True,
+    reset_responses: bool = True,
+    thumbnails: bool = True,
+    xml_metadata: bool = True,
+) -> None:
+    def remove_hidden(cont_lines):
+        """Remove hidden text from a PDF page.
+
+        Args:
+            cont_lines: list of lines with /Contents content. Should have status
+                from after page.cleanContents().
+
+        Returns:
+            List of /Contents lines from which hidden text has been removed.
+
+        Notes:
+            The input must have been created after the page's /Contents object(s)
+            have been cleaned with page.cleanContents(). This ensures a standard
+            formatting: one command per line, single spaces between operators.
+            This allows for drastic simplification of this code.
+        """
+        out_lines = []  # will return this
+        in_text = False  # indicate if within BT/ET object
+        suppress = False  # indicate text suppression active
+        make_return = False
+        for line in cont_lines:
+            if line == b"BT":  # start of text object
+                in_text = True  # switch on
+                out_lines.append(line)  # output it
+                continue
+            if line == b"ET":  # end of text object
+                in_text = False  # switch off
+                out_lines.append(line)  # output it
+                continue
+            if line == b"3 Tr":  # text suppression operator
+                suppress = True  # switch on
+                make_return = True
+                continue
+            if line[-2:] == b"Tr" and line[0] != b"3":
+                suppress = False  # text rendering changed
+                out_lines.append(line)
+                continue
+            if line == b"Q":  # unstack command also switches off
+                suppress = False
+                out_lines.append(line)
+                continue
+            if suppress and in_text:  # suppress hidden lines
+                continue
+            out_lines.append(line)
+        if make_return:
+            return out_lines
+        else:
+            return None
+
+    if not doc.is_pdf:  # only works for PDF
+        raise ValueError("is no PDF")
+    if doc.is_encrypted or doc.is_closed:
+        raise ValueError("closed or encrypted doc")
+
+    if not clean_pages:
+        hidden_text = False
+        redactions = False
+
+    if metadata:
+        doc.set_metadata({})  # remove standard metadata
+
+    for page in doc:
+        if reset_fields:
+            # reset form fields (widgets)
+            for widget in page.widgets():
+                widget.reset()
+
+        if remove_links:
+            links = page.get_links()  # list of all links on page
+            for link in links:  # remove all links
+                page.delete_link(link)
+
+        found_redacts = False
+        for annot in page.annots():
+            if annot.type[0] == mupdf.PDF_ANNOT_FILE_ATTACHMENT and attached_files:
+                annot.update_file(buffer=b" ")  # set file content to empty
+            if reset_responses:
+                annot.delete_responses()
+            if annot.type[0] == pymupdf.PDF_ANNOT_REDACT:  # pylint: disable=no-member
+                found_redacts = True
+
+        if redactions and found_redacts:
+            page.apply_redactions(images=redact_images)
+
+        if not (clean_pages or hidden_text):
+            continue  # done with the page
+
+        page.clean_contents()
+        if not page.get_contents():
+            continue
+        if hidden_text:
+            xref = page.get_contents()[0]  # only one b/o cleaning!
+            cont = doc.xref_stream(xref)
+            cont_lines = remove_hidden(cont.splitlines())  # remove hidden text
+            if cont_lines:  # something was actually removed
+                cont = b"\n".join(cont_lines)
+                doc.update_stream(xref, cont)  # rewrite the page /Contents
+
+        if thumbnails:  # remove page thumbnails?
+            if doc.xref_get_key(page.xref, "Thumb")[0] != "null":
+                doc.xref_set_key(page.xref, "Thumb", "null")
+
+    # pages are scrubbed, now perform document-wide scrubbing
+    # remove embedded files
+    if embedded_files:
+        for name in doc.embfile_names():
+            doc.embfile_del(name)
+
+    if xml_metadata:
+        doc.del_xml_metadata()
+    if not (xml_metadata or javascript):
+        xref_limit = 0
+    else:
+        xref_limit = doc.xref_length()
+    for xref in range(1, xref_limit):
+        if not doc.xref_object(xref):
+            msg = "bad xref %i - clean PDF before scrubbing" % xref
+            raise ValueError(msg)
+        if javascript and doc.xref_get_key(xref, "S")[1] == "/JavaScript":
+            # a /JavaScript action object
+            obj = "<>"  # replace with a null JavaScript
+            doc.update_object(xref, obj)  # update this object
+            continue  # no further handling
+
+        if not xml_metadata:
+            continue
+
+        if doc.xref_get_key(xref, "Type")[1] == "/Metadata":
+            # delete any metadata object directly
+            doc.update_object(xref, "<<>>")
+            doc.update_stream(xref, b"deleted", new=True)
+            continue
+
+        if doc.xref_get_key(xref, "Metadata")[0] != "null":
+            doc.xref_set_key(xref, "Metadata", "null")
+
+
+def _show_fz_text( text):
+    #if mupdf_cppyy:
+    #    assert isinstance( text, cppyy.gbl.mupdf.Text)
+    #else:
+    #    assert isinstance( text, mupdf.Text)
+    num_spans = 0
+    num_chars = 0
+    span = text.m_internal.head
+    while 1:
+        if not span:
+            break
+        num_spans += 1
+        num_chars += span.len
+        span = span.next
+    return f'num_spans={num_spans} num_chars={num_chars}'
+
+def fill_textbox(
+    writer: pymupdf.TextWriter,
+    rect: rect_like,
+    text: typing.Union[str, list],
+    pos: point_like = None,
+    font: typing.Optional[pymupdf.Font] = None,
+    fontsize: float = 11,
+    lineheight: OptFloat = None,
+    align: int = 0,
+    warn: bool = None,
+    right_to_left: bool = False,
+    small_caps: bool = False,
+) -> tuple:
+    """Fill a rectangle with text.
+
+    Args:
+        writer: pymupdf.TextWriter object (= "self")
+        rect: rect-like to receive the text.
+        text: string or list/tuple of strings.
+        pos: point-like start position of first word.
+        font: pymupdf.Font object (default pymupdf.Font('helv')).
+        fontsize: the fontsize.
+        lineheight: overwrite the font property
+        align: (int) 0 = left, 1 = center, 2 = right, 3 = justify
+        warn: (bool) text overflow action: none, warn, or exception
+        right_to_left: (bool) indicate right-to-left language.
+    """
+    rect = pymupdf.Rect(rect)
+    if rect.is_empty:
+        raise ValueError("fill rect must not empty.")
+    if type(font) is not pymupdf.Font:
+        font = pymupdf.Font("helv")
+
+    def textlen(x):
+        """Return length of a string."""
+        return font.text_length(
+            x, fontsize=fontsize, small_caps=small_caps
+        )  # abbreviation
+
+    def char_lengths(x):
+        """Return list of single character lengths for a string."""
+        return font.char_lengths(x, fontsize=fontsize, small_caps=small_caps)
+
+    def append_this(pos, text):
+        ret = writer.append(
+                pos, text, font=font, fontsize=fontsize, small_caps=small_caps
+                )
+        return ret
+
+    tolerance = fontsize * 0.2  # extra distance to left border
+    space_len = textlen(" ")
+    std_width = rect.width - tolerance
+    std_start = rect.x0 + tolerance
+
+    def norm_words(width, words):
+        """Cut any word in pieces no longer than 'width'."""
+        nwords = []
+        word_lengths = []
+        for w in words:
+            wl_lst = char_lengths(w)
+            wl = sum(wl_lst)
+            if wl <= width:  # nothing to do - copy over
+                nwords.append(w)
+                word_lengths.append(wl)
+                continue
+
+            # word longer than rect width - split it in parts
+            n = len(wl_lst)
+            while n > 0:
+                wl = sum(wl_lst[:n])
+                if wl <= width:
+                    nwords.append(w[:n])
+                    word_lengths.append(wl)
+                    w = w[n:]
+                    wl_lst = wl_lst[n:]
+                    n = len(wl_lst)
+                else:
+                    n -= 1
+        return nwords, word_lengths
+
+    def output_justify(start, line):
+        """Justified output of a line."""
+        # ignore leading / trailing / multiple spaces
+        words = [w for w in line.split(" ") if w != ""]
+        nwords = len(words)
+        if nwords == 0:
+            return
+        if nwords == 1:  # single word cannot be justified
+            append_this(start, words[0])
+            return
+        tl = sum([textlen(w) for w in words])  # total word lengths
+        gaps = nwords - 1  # number of word gaps
+        gapl = (std_width - tl) / gaps  # width of each gap
+        for w in words:
+            _, lp = append_this(start, w)  # output one word
+            start.x = lp.x + gapl  # next start at word end plus gap
+        return
+
+    asc = font.ascender
+    dsc = font.descender
+    if not lineheight:
+        if asc - dsc <= 1:
+            lheight = 1.2
+        else:
+            lheight = asc - dsc
+    else:
+        lheight = lineheight
+
+    LINEHEIGHT = fontsize * lheight  # effective line height
+    width = std_width  # available horizontal space
+
+    # starting point of text
+    if pos is not None:
+        pos = pymupdf.Point(pos)
+    else:  # default is just below rect top-left
+        pos = rect.tl + (tolerance, fontsize * asc)
+    if pos not in rect:
+        raise ValueError("Text must start in rectangle.")
+
+    # calculate displacement factor for alignment
+    if align == pymupdf.TEXT_ALIGN_CENTER:
+        factor = 0.5
+    elif align == pymupdf.TEXT_ALIGN_RIGHT:
+        factor = 1.0
+    else:
+        factor = 0
+
+    # split in lines if just a string was given
+    if type(text) is str:
+        textlines = text.splitlines()
+    else:
+        textlines = []
+        for line in text:
+            textlines.extend(line.splitlines())
+
+    max_lines = int((rect.y1 - pos.y) / LINEHEIGHT) + 1
+
+    new_lines = []  # the final list of textbox lines
+    no_justify = []  # no justify for these line numbers
+    for i, line in enumerate(textlines):
+        if line in ("", " "):
+            new_lines.append((line, space_len))
+            width = rect.width - tolerance
+            no_justify.append((len(new_lines) - 1))
+            continue
+        if i == 0:
+            width = rect.x1 - pos.x
+        else:
+            width = rect.width - tolerance
+
+        if right_to_left:  # reverses Arabic / Hebrew text front to back
+            line = writer.clean_rtl(line)
+        tl = textlen(line)
+        if tl <= width:  # line short enough
+            new_lines.append((line, tl))
+            no_justify.append((len(new_lines) - 1))
+            continue
+
+        # we need to split the line in fitting parts
+        words = line.split(" ")  # the words in the line
+
+        # cut in parts any words that are longer than rect width
+        words, word_lengths = norm_words(width, words)
+
+        n = len(words)
+        while True:
+            line0 = " ".join(words[:n])
+            wl = sum(word_lengths[:n]) + space_len * (n - 1)
+            if wl <= width:
+                new_lines.append((line0, wl))
+                words = words[n:]
+                word_lengths = word_lengths[n:]
+                n = len(words)
+                line0 = None
+            else:
+                n -= 1
+
+            if len(words) == 0:
+                break
+            assert n
+
+    # -------------------------------------------------------------------------
+    # List of lines created. Each item is (text, tl), where 'tl' is the PDF
+    # output length (float) and 'text' is the text. Except for justified text,
+    # this is output-ready.
+    # -------------------------------------------------------------------------
+    nlines = len(new_lines)
+    if nlines > max_lines:
+        msg = "Only fitting %i of %i lines." % (max_lines, nlines)
+        if warn is None:
+            pass
+        elif warn:
+            pymupdf.message("Warning: " + msg)
+        else:
+            raise ValueError(msg)
+
+    start = pymupdf.Point()
+    no_justify += [len(new_lines) - 1]  # no justifying of last line
+    for i in range(max_lines):
+        try:
+            line, tl = new_lines.pop(0)
+        except IndexError:
+            if g_exceptions_verbose >= 2:   pymupdf.exception_info()
+            break
+
+        if right_to_left:  # Arabic, Hebrew
+            line = "".join(reversed(line))
+
+        if i == 0:  # may have different start for first line
+            start = pos
+
+        if align == pymupdf.TEXT_ALIGN_JUSTIFY and i not in no_justify and tl < std_width:
+            output_justify(start, line)
+            start.x = std_start
+            start.y += LINEHEIGHT
+            continue
+
+        if i > 0 or pos.x == std_start:  # left, center, right alignments
+            start.x += (width - tl) * factor
+
+        append_this(start, line)
+        start.x = std_start
+        start.y += LINEHEIGHT
+
+    return new_lines  # return non-written lines
+
+
+# ------------------------------------------------------------------------
+# Optional Content functions
+# ------------------------------------------------------------------------
+def get_oc(doc: pymupdf.Document, xref: int) -> int:
+    """Return optional content object xref for an image or form xobject.
+
+    Args:
+        xref: (int) xref number of an image or form xobject.
+    """
+    if doc.is_closed or doc.is_encrypted:
+        raise ValueError("document close or encrypted")
+    t, name = doc.xref_get_key(xref, "Subtype")
+    if t != "name" or name not in ("/Image", "/Form"):
+        raise ValueError("bad object type at xref %i" % xref)
+    t, oc = doc.xref_get_key(xref, "OC")
+    if t != "xref":
+        return 0
+    rc = int(oc.replace("0 R", ""))
+    return rc
+
+
+def set_oc(doc: pymupdf.Document, xref: int, oc: int) -> None:
+    """Attach optional content object to image or form xobject.
+
+    Args:
+        xref: (int) xref number of an image or form xobject
+        oc: (int) xref number of an OCG or OCMD
+    """
+    if doc.is_closed or doc.is_encrypted:
+        raise ValueError("document close or encrypted")
+    t, name = doc.xref_get_key(xref, "Subtype")
+    if t != "name" or name not in ("/Image", "/Form"):
+        raise ValueError("bad object type at xref %i" % xref)
+    if oc > 0:
+        t, name = doc.xref_get_key(oc, "Type")
+        if t != "name" or name not in ("/OCG", "/OCMD"):
+            raise ValueError("bad object type at xref %i" % oc)
+    if oc == 0 and "OC" in doc.xref_get_keys(xref):
+        doc.xref_set_key(xref, "OC", "null")
+        return None
+    doc.xref_set_key(xref, "OC", "%i 0 R" % oc)
+    return None
+
+
+def set_ocmd(
+    doc: pymupdf.Document,
+    xref: int = 0,
+    ocgs: typing.Union[list, None] = None,
+    policy: OptStr = None,
+    ve: typing.Union[list, None] = None,
+) -> int:
+    """Create or update an OCMD object in a PDF document.
+
+    Args:
+        xref: (int) 0 for creating a new object, otherwise update existing one.
+        ocgs: (list) OCG xref numbers, which shall be subject to 'policy'.
+        policy: one of 'AllOn', 'AllOff', 'AnyOn', 'AnyOff' (any casing).
+        ve: (list) visibility expression. Use instead of 'ocgs' with 'policy'.
+
+    Returns:
+        Xref of the created or updated OCMD.
+    """
+
+    all_ocgs = set(doc.get_ocgs().keys())
+
+    def ve_maker(ve):
+        if type(ve) not in (list, tuple) or len(ve) < 2:
+            raise ValueError("bad 've' format: %s" % ve)
+        if ve[0].lower() not in ("and", "or", "not"):
+            raise ValueError("bad operand: %s" % ve[0])
+        if ve[0].lower() == "not" and len(ve) != 2:
+            raise ValueError("bad 've' format: %s" % ve)
+        item = "[/%s" % ve[0].title()
+        for x in ve[1:]:
+            if type(x) is int:
+                if x not in all_ocgs:
+                    raise ValueError("bad OCG %i" % x)
+                item += " %i 0 R" % x
+            else:
+                item += " %s" % ve_maker(x)
+        item += "]"
+        return item
+
+    text = "< dict:
+    """Return the definition of an OCMD (optional content membership dictionary).
+
+    Recognizes PDF dict keys /OCGs (PDF array of OCGs), /P (policy string) and
+    /VE (visibility expression, PDF array). Via string manipulation, this
+    info is converted to a Python dictionary with keys "xref", "ocgs", "policy"
+    and "ve" - ready to recycle as input for 'set_ocmd()'.
+    """
+
+    if xref not in range(doc.xref_length()):
+        raise ValueError("bad xref")
+    text = doc.xref_object(xref, compressed=True)
+    if "/Type/OCMD" not in text:
+        raise ValueError("bad object type")
+    textlen = len(text)
+
+    p0 = text.find("/OCGs[")  # look for /OCGs key
+    p1 = text.find("]", p0)
+    if p0 < 0 or p1 < 0:  # no OCGs found
+        ocgs = None
+    else:
+        ocgs = text[p0 + 6 : p1].replace("0 R", " ").split()
+        ocgs = list(map(int, ocgs))
+
+    p0 = text.find("/P/")  # look for /P policy key
+    if p0 < 0:
+        policy = None
+    else:
+        p1 = text.find("ff", p0)
+        if p1 < 0:
+            p1 = text.find("on", p0)
+        if p1 < 0:  # some irregular syntax
+            raise ValueError("bad object at xref")
+        else:
+            policy = text[p0 + 3 : p1 + 2]
+
+    p0 = text.find("/VE[")  # look for /VE visibility expression key
+    if p0 < 0:  # no visibility expression found
+        ve = None
+    else:
+        lp = rp = 0  # find end of /VE by finding last ']'.
+        p1 = p0
+        while lp < 1 or lp != rp:
+            p1 += 1
+            if not p1 < textlen:  # some irregular syntax
+                raise ValueError("bad object at xref")
+            if text[p1] == "[":
+                lp += 1
+            if text[p1] == "]":
+                rp += 1
+        # p1 now positioned at the last "]"
+        ve = text[p0 + 3 : p1 + 1]  # the PDF /VE array
+        ve = (
+            ve.replace("/And", '"and",')
+            .replace("/Not", '"not",')
+            .replace("/Or", '"or",')
+        )
+        ve = ve.replace(" 0 R]", "]").replace(" 0 R", ",").replace("][", "],[")
+        import json
+        try:
+            ve = json.loads(ve)
+        except Exception:
+            pymupdf.exception_info()
+            pymupdf.message(f"bad /VE key: {ve!r}")
+            raise
+    return {"xref": xref, "ocgs": ocgs, "policy": policy, "ve": ve}
+
+
+"""
+Handle page labels for PDF documents.
+
+Reading
+-------
+* compute the label of a page
+* find page number(s) having the given label.
+
+Writing
+-------
+Supports setting (defining) page labels for PDF documents.
+
+A big Thank You goes to WILLIAM CHAPMAN who contributed the idea and
+significant parts of the following code during late December 2020
+through early January 2021.
+"""
+
+
+def rule_dict(item):
+    """Make a Python dict from a PDF page label rule.
+
+    Args:
+        item -- a tuple (pno, rule) with the start page number and the rule
+                string like <>.
+    Returns:
+        A dict like
+        {'startpage': int, 'prefix': str, 'style': str, 'firstpagenum': int}.
+    """
+    # Jorj McKie, 2021-01-06
+
+    pno, rule = item
+    rule = rule[2:-2].split("/")[1:]  # strip "<<" and ">>"
+    d = {"startpage": pno, "prefix": "", "firstpagenum": 1}
+    skip = False
+    for i, item in enumerate(rule): # pylint: disable=redefined-argument-from-local
+        if skip:  # this item has already been processed
+            skip = False  # deactivate skipping again
+            continue
+        if item == "S":  # style specification
+            d["style"] = rule[i + 1]  # next item has the style
+            skip = True  # do not process next item again
+            continue
+        if item.startswith("P"):  # prefix specification: extract the string
+            x = item[1:].replace("(", "").replace(")", "")
+            d["prefix"] = x
+            continue
+        if item.startswith("St"):  # start page number specification
+            x = int(item[2:])
+            d["firstpagenum"] = x
+    return d
+
+
+def get_label_pno(pgNo, labels):
+    """Return the label for this page number.
+
+    Args:
+        pgNo: page number, 0-based.
+        labels: result of doc._get_page_labels().
+    Returns:
+        The label (str) of the page number. Errors return an empty string.
+    """
+    # Jorj McKie, 2021-01-06
+
+    item = [x for x in labels if x[0] <= pgNo][-1]
+    rule = rule_dict(item)
+    prefix = rule.get("prefix", "")
+    style = rule.get("style", "")
+    # make sure we start at 0 when enumerating the alphabet
+    delta = -1 if style in ("a", "A") else 0
+    pagenumber = pgNo - rule["startpage"] + rule["firstpagenum"] + delta
+    return construct_label(style, prefix, pagenumber)
+
+
+def get_label(page):
+    """Return the label for this PDF page.
+
+    Args:
+        page: page object.
+    Returns:
+        The label (str) of the page. Errors return an empty string.
+    """
+    # Jorj McKie, 2021-01-06
+
+    labels = page.parent._get_page_labels()
+    if not labels:
+        return ""
+    labels.sort()
+    return get_label_pno(page.number, labels)
+
+
+def get_page_numbers(doc, label, only_one=False):
+    """Return a list of page numbers with the given label.
+
+    Args:
+        doc: PDF document object (resp. 'self').
+        label: (str) label.
+        only_one: (bool) stop searching after first hit.
+    Returns:
+        List of page numbers having this label.
+    """
+    # Jorj McKie, 2021-01-06
+
+    numbers = []
+    if not label:
+        return numbers
+    labels = doc._get_page_labels()
+    if labels == []:
+        return numbers
+    for i in range(doc.page_count):
+        plabel = get_label_pno(i, labels)
+        if plabel == label:
+            numbers.append(i)
+            if only_one:
+                break
+    return numbers
+
+
+def construct_label(style, prefix, pno) -> str:
+    """Construct a label based on style, prefix and page number."""
+    # William Chapman, 2021-01-06
+
+    n_str = ""
+    if style == "D":
+        n_str = str(pno)
+    elif style == "r":
+        n_str = integerToRoman(pno).lower()
+    elif style == "R":
+        n_str = integerToRoman(pno).upper()
+    elif style == "a":
+        n_str = integerToLetter(pno).lower()
+    elif style == "A":
+        n_str = integerToLetter(pno).upper()
+    result = prefix + n_str
+    return result
+
+
+def integerToLetter(i) -> str:
+    """Returns letter sequence string for integer i."""
+    # William Chapman, Jorj McKie, 2021-01-06
+    import string
+    ls = string.ascii_uppercase
+    n, a = 1, i
+    while pow(26, n) <= a:
+        a -= int(math.pow(26, n))
+        n += 1
+
+    str_t = ""
+    for j in reversed(range(n)):
+        f, g = divmod(a, int(math.pow(26, j)))
+        str_t += ls[f]
+        a = g
+    return str_t
+
+
+def integerToRoman(num: int) -> str:
+    """Return roman numeral for an integer."""
+    # William Chapman, Jorj McKie, 2021-01-06
+
+    roman = (
+        (1000, "M"),
+        (900, "CM"),
+        (500, "D"),
+        (400, "CD"),
+        (100, "C"),
+        (90, "XC"),
+        (50, "L"),
+        (40, "XL"),
+        (10, "X"),
+        (9, "IX"),
+        (5, "V"),
+        (4, "IV"),
+        (1, "I"),
+    )
+
+    def roman_num(num):
+        for r, ltr in roman:
+            x, _ = divmod(num, r)
+            yield ltr * x
+            num -= r * x
+            if num <= 0:
+                break
+
+    return "".join([a for a in roman_num(num)])
+
+
+def get_page_labels(doc):
+    """Return page label definitions in PDF document.
+
+    Args:
+        doc: PDF document (resp. 'self').
+    Returns:
+        A list of dictionaries with the following format:
+        {'startpage': int, 'prefix': str, 'style': str, 'firstpagenum': int}.
+    """
+    # Jorj McKie, 2021-01-10
+    return [rule_dict(item) for item in doc._get_page_labels()]
+
+
+def set_page_labels(doc, labels):
+    """Add / replace page label definitions in PDF document.
+
+    Args:
+        doc: PDF document (resp. 'self').
+        labels: list of label dictionaries like:
+        {'startpage': int, 'prefix': str, 'style': str, 'firstpagenum': int},
+        as returned by get_page_labels().
+    """
+    # William Chapman, 2021-01-06
+
+    def create_label_str(label):
+        """Convert Python label dict to corresponding PDF rule string.
+
+        Args:
+            label: (dict) build rule for the label.
+        Returns:
+            PDF label rule string wrapped in "<<", ">>".
+        """
+        s = "%i<<" % label["startpage"]
+        if label.get("prefix", "") != "":
+            s += "/P(%s)" % label["prefix"]
+        if label.get("style", "") != "":
+            s += "/S/%s" % label["style"]
+        if label.get("firstpagenum", 1) > 1:
+            s += "/St %i" % label["firstpagenum"]
+        s += ">>"
+        return s
+
+    def create_nums(labels):
+        """Return concatenated string of all labels rules.
+
+        Args:
+            labels: (list) dictionaries as created by function 'rule_dict'.
+        Returns:
+            PDF compatible string for page label definitions, ready to be
+            enclosed in PDF array 'Nums[...]'.
+        """
+        labels.sort(key=lambda x: x["startpage"])
+        s = "".join([create_label_str(label) for label in labels])
+        return s
+
+    doc._set_page_labels(create_nums(labels))
+
+
+# End of Page Label Code -------------------------------------------------
+
+
+def has_links(doc: pymupdf.Document) -> bool:
+    """Check whether there are links on any page."""
+    if doc.is_closed:
+        raise ValueError("document closed")
+    if not doc.is_pdf:
+        raise ValueError("is no PDF")
+    for i in range(doc.page_count):
+        for item in doc.page_annot_xrefs(i):
+            if item[1] == pymupdf.PDF_ANNOT_LINK:  # pylint: disable=no-member
+                return True
+    return False
+
+
+def has_annots(doc: pymupdf.Document) -> bool:
+    """Check whether there are annotations on any page."""
+    if doc.is_closed:
+        raise ValueError("document closed")
+    if not doc.is_pdf:
+        raise ValueError("is no PDF")
+    for i in range(doc.page_count):
+        for item in doc.page_annot_xrefs(i):
+            # pylint: disable=no-member
+            if not (item[1] == pymupdf.PDF_ANNOT_LINK or item[1] == pymupdf.PDF_ANNOT_WIDGET):  # pylint: disable=no-member
+                return True
+    return False
+
+
+# -------------------------------------------------------------------
+# Functions to recover the quad contained in a text extraction bbox
+# -------------------------------------------------------------------
+def recover_bbox_quad(line_dir: tuple, span: dict, bbox: tuple) -> pymupdf.Quad:
+    """Compute the quad located inside the bbox.
+
+    The bbox may be any of the resp. tuples occurring inside the given span.
+
+    Args:
+        line_dir: (tuple) 'line["dir"]' of the owning line or None.
+        span: (dict) the span. May be from get_texttrace() method.
+        bbox: (tuple) the bbox of the span or any of its characters.
+    Returns:
+        The quad which is wrapped by the bbox.
+    """
+    if line_dir is None:
+        line_dir = span["dir"]
+    cos, sin = line_dir
+    bbox = pymupdf.Rect(bbox)  # make it a rect
+    if pymupdf.TOOLS.set_small_glyph_heights():  # ==> just fontsize as height
+        d = 1
+    else:
+        d = span["ascender"] - span["descender"]
+
+    height = d * span["size"]  # the quad's rectangle height
+    # The following are distances from the bbox corners, at which we find the
+    # respective quad points. The computation depends on in which quadrant the
+    # text writing angle is located.
+    hs = height * sin
+    hc = height * cos
+    if hc >= 0 and hs <= 0:  # quadrant 1
+        ul = bbox.bl - (0, hc)
+        ur = bbox.tr + (hs, 0)
+        ll = bbox.bl - (hs, 0)
+        lr = bbox.tr + (0, hc)
+    elif hc <= 0 and hs <= 0:  # quadrant 2
+        ul = bbox.br + (hs, 0)
+        ur = bbox.tl - (0, hc)
+        ll = bbox.br + (0, hc)
+        lr = bbox.tl - (hs, 0)
+    elif hc <= 0 and hs >= 0:  # quadrant 3
+        ul = bbox.tr - (0, hc)
+        ur = bbox.bl + (hs, 0)
+        ll = bbox.tr - (hs, 0)
+        lr = bbox.bl + (0, hc)
+    else:  # quadrant 4
+        ul = bbox.tl + (hs, 0)
+        ur = bbox.br - (0, hc)
+        ll = bbox.tl + (0, hc)
+        lr = bbox.br - (hs, 0)
+    return pymupdf.Quad(ul, ur, ll, lr)
+
+
+def recover_quad(line_dir: tuple, span: dict) -> pymupdf.Quad:
+    """Recover the quadrilateral of a text span.
+
+    Args:
+        line_dir: (tuple) 'line["dir"]' of the owning line.
+        span: the span.
+    Returns:
+        The quadrilateral enveloping the span's text.
+    """
+    if type(line_dir) is not tuple or len(line_dir) != 2:
+        raise ValueError("bad line dir argument")
+    if type(span) is not dict:
+        raise ValueError("bad span argument")
+    return recover_bbox_quad(line_dir, span, span["bbox"])
+
+
+def recover_line_quad(line: dict, spans: list = None) -> pymupdf.Quad:
+    """Calculate the line quad for 'dict' / 'rawdict' text extractions.
+
+    The lower quad points are those of the first, resp. last span quad.
+    The upper points are determined by the maximum span quad height.
+    From this, compute a rect with bottom-left in (0, 0), convert this to a
+    quad and rotate and shift back to cover the text of the spans.
+
+    Args:
+        spans: (list, optional) sub-list of spans to consider.
+    Returns:
+        pymupdf.Quad covering selected spans.
+    """
+    if spans is None:  # no sub-selection
+        spans = line["spans"]  # all spans
+    if len(spans) == 0:
+        raise ValueError("bad span list")
+    line_dir = line["dir"]  # text direction
+    cos, sin = line_dir
+    q0 = recover_quad(line_dir, spans[0])  # quad of first span
+    if len(spans) > 1:  # get quad of last span
+        q1 = recover_quad(line_dir, spans[-1])
+    else:
+        q1 = q0  # last = first
+
+    line_ll = q0.ll  # lower-left of line quad
+    line_lr = q1.lr  # lower-right of line quad
+
+    mat0 = pymupdf.planish_line(line_ll, line_lr)
+
+    # map base line to x-axis such that line_ll goes to (0, 0)
+    x_lr = line_lr * mat0
+
+    small = pymupdf.TOOLS.set_small_glyph_heights()  # small glyph heights?
+
+    h = max(
+        [s["size"] * (1 if small else (s["ascender"] - s["descender"])) for s in spans]
+    )
+
+    line_rect = pymupdf.Rect(0, -h, x_lr.x, 0)  # line rectangle
+    line_quad = line_rect.quad  # make it a quad and:
+    line_quad *= ~mat0
+    return line_quad
+
+
+def recover_span_quad(line_dir: tuple, span: dict, chars: list = None) -> pymupdf.Quad:
+    """Calculate the span quad for 'dict' / 'rawdict' text extractions.
+
+    Notes:
+        There are two execution paths:
+        1. For the full span quad, the result of 'recover_quad' is returned.
+        2. For the quad of a sub-list of characters, the char quads are
+           computed and joined. This is only supported for the "rawdict"
+           extraction option.
+
+    Args:
+        line_dir: (tuple) 'line["dir"]' of the owning line.
+        span: (dict) the span.
+        chars: (list, optional) sub-list of characters to consider.
+    Returns:
+        pymupdf.Quad covering selected characters.
+    """
+    if line_dir is None:  # must be a span from get_texttrace()
+        line_dir = span["dir"]
+    if chars is None:  # no sub-selection
+        return recover_quad(line_dir, span)
+    if "chars" not in span.keys():
+        raise ValueError("need 'rawdict' option to sub-select chars")
+
+    q0 = recover_char_quad(line_dir, span, chars[0])  # quad of first char
+    if len(chars) > 1:  # get quad of last char
+        q1 = recover_char_quad(line_dir, span, chars[-1])
+    else:
+        q1 = q0  # last = first
+
+    span_ll = q0.ll  # lower-left of span quad
+    span_lr = q1.lr  # lower-right of span quad
+    mat0 = pymupdf.planish_line(span_ll, span_lr)
+    # map base line to x-axis such that span_ll goes to (0, 0)
+    x_lr = span_lr * mat0
+
+    small = pymupdf.TOOLS.set_small_glyph_heights()  # small glyph heights?
+    h = span["size"] * (1 if small else (span["ascender"] - span["descender"]))
+
+    span_rect = pymupdf.Rect(0, -h, x_lr.x, 0)  # line rectangle
+    span_quad = span_rect.quad  # make it a quad and:
+    span_quad *= ~mat0  # rotate back and shift back
+    return span_quad
+
+
+def recover_char_quad(line_dir: tuple, span: dict, char: dict) -> pymupdf.Quad:
+    """Recover the quadrilateral of a text character.
+
+    This requires the "rawdict" option of text extraction.
+
+    Args:
+        line_dir: (tuple) 'line["dir"]' of the span's line.
+        span: (dict) the span dict.
+        char: (dict) the character dict.
+    Returns:
+        The quadrilateral enveloping the character.
+    """
+    if line_dir is None:
+        line_dir = span["dir"]
+    if type(line_dir) is not tuple or len(line_dir) != 2:
+        raise ValueError("bad line dir argument")
+    if type(span) is not dict:
+        raise ValueError("bad span argument")
+    if type(char) is dict:
+        bbox = pymupdf.Rect(char["bbox"])
+    elif type(char) is tuple:
+        bbox = pymupdf.Rect(char[3])
+    else:
+        raise ValueError("bad span argument")
+
+    return recover_bbox_quad(line_dir, span, bbox)
+
+
+# -------------------------------------------------------------------
+# Building font subsets using fontTools
+# -------------------------------------------------------------------
+def subset_fonts(doc: pymupdf.Document, verbose: bool = False, fallback: bool = False) -> OptInt:
+    """Build font subsets in a PDF.
+
+    Eligible fonts are potentially replaced by smaller versions. Page text is
+    NOT rewritten and thus should retain properties like being hidden or
+    controlled by optional content.
+
+    This method by default uses MuPDF's own internal feature to create subset
+    fonts. As this is a new function, errors may still occur. In this case,
+    please fall back to using the previous version by using "fallback=True".
+    Fallback mode requires the external package 'fontTools'.
+
+    Args:
+        fallback: use the older deprecated implementation.
+        verbose: only used by fallback mode.
+
+    Returns:
+        The new MuPDF-based code returns None.  The deprecated fallback
+        mode returns 0 if there are no fonts to subset.  Otherwise, it
+        returns the decrease in fontsize (the difference in fontsize),
+        measured in bytes.
+    """
+    # Font binaries: -  "buffer" -> (names, xrefs, (unicodes, glyphs))
+    # An embedded font is uniquely defined by its fontbuffer only. It may have
+    # multiple names and xrefs.
+    # Once the sets of used unicodes and glyphs are known, we compute a
+    # smaller version of the buffer user package fontTools.
+
+    if not fallback:  # by default use MuPDF function
+        pdf = mupdf.pdf_document_from_fz_document(doc)
+        mupdf.pdf_subset_fonts2(pdf, list(range(doc.page_count)))
+        return
+
+    font_buffers = {}
+
+    def get_old_widths(xref):
+        """Retrieve old font '/W' and '/DW' values."""
+        df = doc.xref_get_key(xref, "DescendantFonts")
+        if df[0] != "array":  # only handle xref specifications
+            return None, None
+        df_xref = int(df[1][1:-1].replace("0 R", ""))
+        widths = doc.xref_get_key(df_xref, "W")
+        if widths[0] != "array":  # no widths key found
+            widths = None
+        else:
+            widths = widths[1]
+        dwidths = doc.xref_get_key(df_xref, "DW")
+        if dwidths[0] != "int":
+            dwidths = None
+        else:
+            dwidths = dwidths[1]
+        return widths, dwidths
+
+    def set_old_widths(xref, widths, dwidths):
+        """Restore the old '/W' and '/DW' in subsetted font.
+
+        If either parameter is None or evaluates to False, the corresponding
+        dictionary key will be set to null.
+        """
+        df = doc.xref_get_key(xref, "DescendantFonts")
+        if df[0] != "array":  # only handle xref specs
+            return None
+        df_xref = int(df[1][1:-1].replace("0 R", ""))
+        if (type(widths) is not str or not widths) and doc.xref_get_key(df_xref, "W")[
+            0
+        ] != "null":
+            doc.xref_set_key(df_xref, "W", "null")
+        else:
+            doc.xref_set_key(df_xref, "W", widths)
+        if (type(dwidths) is not str or not dwidths) and doc.xref_get_key(
+            df_xref, "DW"
+        )[0] != "null":
+            doc.xref_set_key(df_xref, "DW", "null")
+        else:
+            doc.xref_set_key(df_xref, "DW", dwidths)
+        return None
+
+    def set_subset_fontname(new_xref):
+        """Generate a name prefix to tag a font as subset.
+
+        We use a random generator to select 6 upper case ASCII characters.
+        The prefixed name must be put in the font xref as the "/BaseFont" value
+        and in the FontDescriptor object as the '/FontName' value.
+        """
+        # The following generates a prefix like 'ABCDEF+'
+        import random
+        import string
+        prefix = "".join(random.choices(tuple(string.ascii_uppercase), k=6)) + "+"
+        font_str = doc.xref_object(new_xref, compressed=True)
+        font_str = font_str.replace("/BaseFont/", "/BaseFont/" + prefix)
+        df = doc.xref_get_key(new_xref, "DescendantFonts")
+        if df[0] == "array":
+            df_xref = int(df[1][1:-1].replace("0 R", ""))
+            fd = doc.xref_get_key(df_xref, "FontDescriptor")
+            if fd[0] == "xref":
+                fd_xref = int(fd[1].replace("0 R", ""))
+                fd_str = doc.xref_object(fd_xref, compressed=True)
+                fd_str = fd_str.replace("/FontName/", "/FontName/" + prefix)
+                doc.update_object(fd_xref, fd_str)
+        doc.update_object(new_xref, font_str)
+
+    def build_subset(buffer, unc_set, gid_set):
+        """Build font subset using fontTools.
+
+        Args:
+            buffer: (bytes) the font given as a binary buffer.
+            unc_set: (set) required glyph ids.
+        Returns:
+            Either None if subsetting is unsuccessful or the subset font buffer.
+        """
+        try:
+            import fontTools.subset as fts
+        except ImportError:
+            if g_exceptions_verbose:    pymupdf.exception_info()
+            pymupdf.message("This method requires fontTools to be installed.")
+            raise
+        import tempfile
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            oldfont_path = f"{tmp_dir}/oldfont.ttf"
+            newfont_path = f"{tmp_dir}/newfont.ttf"
+            uncfile_path = f"{tmp_dir}/uncfile.txt"
+            args = [
+                oldfont_path,
+                "--retain-gids",
+                f"--output-file={newfont_path}",
+                "--layout-features=*",
+                "--passthrough-tables",
+                "--ignore-missing-glyphs",
+                "--ignore-missing-unicodes",
+                "--symbol-cmap",
+            ]
+
+            # store glyph ids or unicodes as file
+            with open(f"{tmp_dir}/uncfile.txt", "w", encoding='utf8') as unc_file:
+                if 0xFFFD in unc_set:  # error unicode exists -> use glyphs
+                    args.append(f"--gids-file={uncfile_path}")
+                    gid_set.add(189)
+                    unc_list = list(gid_set)
+                    for unc in unc_list:
+                        unc_file.write("%i\n" % unc)
+                else:
+                    args.append(f"--unicodes-file={uncfile_path}")
+                    unc_set.add(255)
+                    unc_list = list(unc_set)
+                    for unc in unc_list:
+                        unc_file.write("%04x\n" % unc)
+
+            # store fontbuffer as a file
+            with open(oldfont_path, "wb") as fontfile:
+                fontfile.write(buffer)
+            try:
+                os.remove(newfont_path)  # remove old file
+            except Exception:
+                pass
+            try:  # invoke fontTools subsetter
+                fts.main(args)
+                font = pymupdf.Font(fontfile=newfont_path)
+                new_buffer = font.buffer  # subset font binary
+                if font.glyph_count == 0:  # intercept empty font
+                    new_buffer = None
+            except Exception:
+                pymupdf.exception_info()
+                new_buffer = None
+        return new_buffer
+
+    def repl_fontnames(doc):
+        """Populate 'font_buffers'.
+
+        For each font candidate, store its xref and the list of names
+        by which PDF text may refer to it (there may be multiple).
+        """
+
+        def norm_name(name):
+            """Recreate font name that contains PDF hex codes.
+
+            E.g. #20 -> space, chr(32)
+            """
+            while "#" in name:
+                p = name.find("#")
+                c = int(name[p + 1 : p + 3], 16)
+                name = name.replace(name[p : p + 3], chr(c))
+            return name
+
+        def get_fontnames(doc, item):
+            """Return a list of fontnames for an item of page.get_fonts().
+
+            There may be multiple names e.g. for Type0 fonts.
+            """
+            fontname = item[3]
+            names = [fontname]
+            fontname = doc.xref_get_key(item[0], "BaseFont")[1][1:]
+            fontname = norm_name(fontname)
+            if fontname not in names:
+                names.append(fontname)
+            descendents = doc.xref_get_key(item[0], "DescendantFonts")
+            if descendents[0] != "array":
+                return names
+            descendents = descendents[1][1:-1]
+            if descendents.endswith(" 0 R"):
+                xref = int(descendents[:-4])
+                descendents = doc.xref_object(xref, compressed=True)
+            p1 = descendents.find("/BaseFont")
+            if p1 >= 0:
+                p2 = descendents.find("/", p1 + 1)
+                p1 = min(descendents.find("/", p2 + 1), descendents.find(">>", p2 + 1))
+                fontname = descendents[p2 + 1 : p1]
+                fontname = norm_name(fontname)
+                if fontname not in names:
+                    names.append(fontname)
+            return names
+
+        for i in range(doc.page_count):
+            for f in doc.get_page_fonts(i, full=True):
+                font_xref = f[0]  # font xref
+                font_ext = f[1]  # font file extension
+                basename = f[3]  # font basename
+
+                if font_ext not in (  # skip if not supported by fontTools
+                    "otf",
+                    "ttf",
+                    "woff",
+                    "woff2",
+                ):
+                    continue
+                # skip fonts which already are subsets
+                if len(basename) > 6 and basename[6] == "+":
+                    continue
+
+                extr = doc.extract_font(font_xref)
+                fontbuffer = extr[-1]
+                names = get_fontnames(doc, f)
+                name_set, xref_set, subsets = font_buffers.get(
+                    fontbuffer, (set(), set(), (set(), set()))
+                )
+                xref_set.add(font_xref)
+                for name in names:
+                    name_set.add(name)
+                font = pymupdf.Font(fontbuffer=fontbuffer)
+                name_set.add(font.name)
+                del font
+                font_buffers[fontbuffer] = (name_set, xref_set, subsets)
+
+    def find_buffer_by_name(name):
+        for buffer, (name_set, _, _) in font_buffers.items():
+            if name in name_set:
+                return buffer
+        return None
+
+    # -----------------
+    # main function
+    # -----------------
+    repl_fontnames(doc)  # populate font information
+    if not font_buffers:  # nothing found to do
+        if verbose:
+            pymupdf.message(f'No fonts to subset.')
+        return 0
+
+    old_fontsize = 0
+    new_fontsize = 0
+    for fontbuffer in font_buffers.keys():
+        old_fontsize += len(fontbuffer)
+
+    # Scan page text for usage of subsettable fonts
+    for page in doc:
+        # go through the text and extend set of used glyphs by font
+        # we use a modified MuPDF trace device, which delivers us glyph ids.
+        for span in page.get_texttrace():
+            if type(span) is not dict:  # skip useless information
+                continue
+            fontname = span["font"][:33]  # fontname for the span
+            buffer = find_buffer_by_name(fontname)
+            if buffer is None:
+                continue
+            name_set, xref_set, (set_ucs, set_gid) = font_buffers[buffer]
+            for c in span["chars"]:
+                set_ucs.add(c[0])  # unicode
+                set_gid.add(c[1])  # glyph id
+            font_buffers[buffer] = (name_set, xref_set, (set_ucs, set_gid))
+
+    # build the font subsets
+    for old_buffer, (name_set, xref_set, subsets) in font_buffers.items():
+        new_buffer = build_subset(old_buffer, subsets[0], subsets[1])
+        fontname = list(name_set)[0]
+        if new_buffer is None or len(new_buffer) >= len(old_buffer):
+            # subset was not created or did not get smaller
+            if verbose:
+                pymupdf.message(f'Cannot subset {fontname!r}.')
+            continue
+        if verbose:
+            pymupdf.message(f"Built subset of font {fontname!r}.")
+        val = doc._insert_font(fontbuffer=new_buffer)  # store subset font in PDF
+        new_xref = val[0]  # get its xref
+        set_subset_fontname(new_xref)  # tag fontname as subset font
+        font_str = doc.xref_object(  # get its object definition
+            new_xref,
+            compressed=True,
+        )
+        # walk through the original font xrefs and replace each by the subset def
+        for font_xref in xref_set:
+            # we need the original '/W' and '/DW' width values
+            width_table, def_width = get_old_widths(font_xref)
+            # ... and replace original font definition at xref with it
+            doc.update_object(font_xref, font_str)
+            # now copy over old '/W' and '/DW' values
+            if width_table or def_width:
+                set_old_widths(font_xref, width_table, def_width)
+        # 'new_xref' remains unused in the PDF and must be removed
+        # by garbage collection.
+        new_fontsize += len(new_buffer)
+
+    return old_fontsize - new_fontsize
+
+
+# -------------------------------------------------------------------
+# Copy XREF object to another XREF
+# -------------------------------------------------------------------
+def xref_copy(doc: pymupdf.Document, source: int, target: int, *, keep: list = None) -> None:
+    """Copy a PDF dictionary object to another one given their xref numbers.
+
+    Args:
+        doc: PDF document object
+        source: source xref number
+        target: target xref number, the xref must already exist
+        keep: an optional list of 1st level keys in target that should not be
+              removed before copying.
+    Notes:
+        This works similar to the copy() method of dictionaries in Python. The
+        source may be a stream object.
+    """
+    if doc.xref_is_stream(source):
+        # read new xref stream, maintaining compression
+        stream = doc.xref_stream_raw(source)
+        doc.update_stream(
+            target,
+            stream,
+            compress=False,  # keeps source compression
+            new=True,  # in case target is no stream
+        )
+
+    # empty the target completely, observe exceptions
+    if keep is None:
+        keep = []
+    for key in doc.xref_get_keys(target):
+        if key in keep:
+            continue
+        doc.xref_set_key(target, key, "null")
+    # copy over all source dict items
+    for key in doc.xref_get_keys(source):
+        item = doc.xref_get_key(source, key)
+        doc.xref_set_key(target, key, item[1])
diff --git a/lib/python3.1/site-packages/pynvml.py b/lib/python3.1/site-packages/pynvml.py
new file mode 100644
index 0000000000000000000000000000000000000000..3603620406c98b94a7a30717e1466acbd027030a
--- /dev/null
+++ b/lib/python3.1/site-packages/pynvml.py
@@ -0,0 +1,6688 @@
+#####
+# Copyright (c) 2011-2025, NVIDIA Corporation.  All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright notice,
+#      this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#    * Neither the name of the NVIDIA Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived from
+#      this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+# THE POSSIBILITY OF SUCH DAMAGE.
+#####
+
+##
+# Python bindings for the NVML library
+##
+from ctypes import *
+from ctypes.util import find_library
+from functools import wraps
+import sys
+import os
+import threading
+import string
+
+## C Type mappings ##
+## Enums
+_nvmlEnableState_t = c_uint
+NVML_FEATURE_DISABLED    = 0
+NVML_FEATURE_ENABLED     = 1
+
+_nvmlBrandType_t = c_uint
+NVML_BRAND_UNKNOWN             = 0
+NVML_BRAND_QUADRO              = 1
+NVML_BRAND_TESLA               = 2
+NVML_BRAND_NVS                 = 3
+NVML_BRAND_GRID                = 4   # Deprecated from API reporting. Keeping definition for backward compatibility.
+NVML_BRAND_GEFORCE             = 5
+NVML_BRAND_TITAN               = 6
+NVML_BRAND_NVIDIA_VAPPS        = 7   # NVIDIA Virtual Applications
+NVML_BRAND_NVIDIA_VPC          = 8   # NVIDIA Virtual PC
+NVML_BRAND_NVIDIA_VCS          = 9   # NVIDIA Virtual Compute Server
+NVML_BRAND_NVIDIA_VWS          = 10  # NVIDIA RTX Virtual Workstation
+NVML_BRAND_NVIDIA_CLOUD_GAMING = 11  # NVIDIA Cloud Gaming
+NVML_BRAND_NVIDIA_VGAMING      = NVML_BRAND_NVIDIA_CLOUD_GAMING # Deprecated from API reporting. Keeping definition for backward compatibility.
+NVML_BRAND_QUADRO_RTX          = 12
+NVML_BRAND_NVIDIA_RTX          = 13
+NVML_BRAND_NVIDIA              = 14
+NVML_BRAND_GEFORCE_RTX         = 15  # Unused
+NVML_BRAND_TITAN_RTX           = 16  # Unused
+NVML_BRAND_COUNT               = 17
+
+_nvmlTemperatureThresholds_t = c_uint
+NVML_TEMPERATURE_THRESHOLD_SHUTDOWN      = 0
+NVML_TEMPERATURE_THRESHOLD_SLOWDOWN      = 1
+NVML_TEMPERATURE_THRESHOLD_MEM_MAX       = 2
+NVML_TEMPERATURE_THRESHOLD_GPU_MAX       = 3
+NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_MIN  = 4
+NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_CURR = 5
+NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_MAX  = 6
+NVML_TEMPERATURE_THRESHOLD_GPS_CURR      = 7
+NVML_TEMPERATURE_THRESHOLD_COUNT         = 8
+
+_nvmlTemperatureSensors_t = c_uint
+NVML_TEMPERATURE_GPU     = 0
+NVML_TEMPERATURE_COUNT   = 1
+
+
+_nvmlComputeMode_t = c_uint
+NVML_COMPUTEMODE_DEFAULT           = 0
+NVML_COMPUTEMODE_EXCLUSIVE_THREAD  = 1  ## Support Removed
+NVML_COMPUTEMODE_PROHIBITED        = 2
+NVML_COMPUTEMODE_EXCLUSIVE_PROCESS = 3
+NVML_COMPUTEMODE_COUNT             = 4
+
+_nvmlMemoryLocation_t = c_uint
+NVML_MEMORY_LOCATION_L1_CACHE = 0
+NVML_MEMORY_LOCATION_L2_CACHE = 1
+NVML_MEMORY_LOCATION_DEVICE_MEMORY = 2
+NVML_MEMORY_LOCATION_DRAM = 2
+NVML_MEMORY_LOCATION_REGISTER_FILE = 3
+NVML_MEMORY_LOCATION_TEXTURE_MEMORY = 4
+NVML_MEMORY_LOCATION_TEXTURE_SHM = 5
+NVML_MEMORY_LOCATION_CBU = 6
+NVML_MEMORY_LOCATION_SRAM = 7
+NVML_MEMORY_LOCATION_COUNT = 8
+
+NVML_NVLINK_MAX_LINKS = 18
+
+# For backwards compatibility, maintain the incorrectly-named "LANES" define
+NVML_NVLINK_MAX_LANES = NVML_NVLINK_MAX_LINKS
+
+_nvmlNvLinkErrorCounter_t = c_uint
+NVML_NVLINK_ERROR_DL_REPLAY = 0
+NVML_NVLINK_ERROR_DL_RECOVERY = 1
+NVML_NVLINK_ERROR_DL_CRC_FLIT = 2
+NVML_NVLINK_ERROR_DL_CRC_DATA = 3
+NVML_NVLINK_ERROR_DL_ECC_DATA = 4
+NVML_NVLINK_ERROR_COUNT = 5
+
+_nvmlNvLinkEccLaneErrorCounter_t = c_uint
+NVML_NVLINK_ERROR_DL_ECC_LANE0 = 0
+NVML_NVLINK_ERROR_DL_ECC_LANE1 = 1
+NVML_NVLINK_ERROR_DL_ECC_LANE2 = 2
+NVML_NVLINK_ERROR_DL_ECC_LANE3 = 3
+NVML_NVLINK_ERROR_DL_ECC_COUNT = 5
+
+_nvmlNvLinkCapability_t = c_uint
+NVML_NVLINK_CAP_P2P_SUPPORTED = 0
+NVML_NVLINK_CAP_SYSMEM_ACCESS = 1
+NVML_NVLINK_CAP_P2P_ATOMICS   = 2
+NVML_NVLINK_CAP_SYSMEM_ATOMICS= 3
+NVML_NVLINK_CAP_SLI_BRIDGE    = 4
+NVML_NVLINK_CAP_VALID         = 5
+NVML_NVLINK_CAP_COUNT         = 6
+
+_nvmlNvLinkUtilizationCountPktTypes_t = c_uint
+NVML_NVLINK_COUNTER_PKTFILTER_NOP        = 0x1
+NVML_NVLINK_COUNTER_PKTFILTER_READ       = 0x2
+NVML_NVLINK_COUNTER_PKTFILTER_WRITE      = 0x4
+NVML_NVLINK_COUNTER_PKTFILTER_RATOM      = 0x8
+NVML_NVLINK_COUNTER_PKTFILTER_NRATOM     = 0x10
+NVML_NVLINK_COUNTER_PKTFILTER_FLUSH      = 0x20
+NVML_NVLINK_COUNTER_PKTFILTER_RESPDATA   = 0x40
+NVML_NVLINK_COUNTER_PKTFILTER_RESPNODATA = 0x80
+NVML_NVLINK_COUNTER_PKTFILTER_ALL        = 0xFF
+
+_nvmlNvLinkUtilizationCountUnits_t = c_uint
+NVML_NVLINK_COUNTER_UNIT_CYCLES   = 0
+NVML_NVLINK_COUNTER_UNIT_PACKETS  = 1
+NVML_NVLINK_COUNTER_UNIT_BYTES    = 2
+NVML_NVLINK_COUNTER_UNIT_RESERVED = 3
+NVML_NVLINK_COUNTER_UNIT_COUNT    = 4
+
+_nvmlNvLinkDeviceType_t = c_uint
+NVML_NVLINK_DEVICE_TYPE_GPU     = 0x00
+NVML_NVLINK_DEVICE_TYPE_IBMNPU  = 0x01
+NVML_NVLINK_DEVICE_TYPE_SWITCH  = 0x02
+NVML_NVLINK_DEVICE_TYPE_UNKNOWN = 0xFF
+
+# These are deprecated, instead use _nvmlMemoryErrorType_t
+_nvmlEccBitType_t = c_uint
+NVML_SINGLE_BIT_ECC    = 0
+NVML_DOUBLE_BIT_ECC    = 1
+NVML_ECC_ERROR_TYPE_COUNT = 2
+
+_nvmlEccCounterType_t = c_uint
+NVML_VOLATILE_ECC      = 0
+NVML_AGGREGATE_ECC     = 1
+NVML_ECC_COUNTER_TYPE_COUNT = 2
+
+_nvmlMemoryErrorType_t = c_uint
+NVML_MEMORY_ERROR_TYPE_CORRECTED   = 0
+NVML_MEMORY_ERROR_TYPE_UNCORRECTED = 1
+NVML_MEMORY_ERROR_TYPE_COUNT       = 2
+
+_nvmlClockType_t = c_uint
+NVML_CLOCK_GRAPHICS  = 0
+NVML_CLOCK_SM        = 1
+NVML_CLOCK_MEM       = 2
+NVML_CLOCK_VIDEO     = 3
+NVML_CLOCK_COUNT     = 4
+
+_nvmlClockId_t = c_uint
+NVML_CLOCK_ID_CURRENT            = 0
+NVML_CLOCK_ID_APP_CLOCK_TARGET   = 1
+NVML_CLOCK_ID_APP_CLOCK_DEFAULT  = 2
+NVML_CLOCK_ID_CUSTOMER_BOOST_MAX = 3
+NVML_CLOCK_ID_COUNT              = 4
+
+_nvmlDriverModel_t = c_uint
+NVML_DRIVER_WDDM       = 0
+NVML_DRIVER_WDM        = 1
+NVML_DRIVER_MCDM       = 2
+
+NVML_MAX_GPU_PERF_PSTATES = 16
+
+_nvmlPstates_t = c_uint
+NVML_PSTATE_0               = 0
+NVML_PSTATE_1               = 1
+NVML_PSTATE_2               = 2
+NVML_PSTATE_3               = 3
+NVML_PSTATE_4               = 4
+NVML_PSTATE_5               = 5
+NVML_PSTATE_6               = 6
+NVML_PSTATE_7               = 7
+NVML_PSTATE_8               = 8
+NVML_PSTATE_9               = 9
+NVML_PSTATE_10              = 10
+NVML_PSTATE_11              = 11
+NVML_PSTATE_12              = 12
+NVML_PSTATE_13              = 13
+NVML_PSTATE_14              = 14
+NVML_PSTATE_15              = 15
+NVML_PSTATE_UNKNOWN         = 32
+
+_nvmlInforomObject_t = c_uint
+NVML_INFOROM_OEM            = 0
+NVML_INFOROM_ECC            = 1
+NVML_INFOROM_POWER          = 2
+NVML_INFOROM_DEN            = 3
+NVML_INFOROM_COUNT          = 4
+
+_nvmlReturn_t = c_uint
+NVML_SUCCESS                         = 0
+NVML_ERROR_UNINITIALIZED             = 1
+NVML_ERROR_INVALID_ARGUMENT          = 2
+NVML_ERROR_NOT_SUPPORTED             = 3
+NVML_ERROR_NO_PERMISSION             = 4
+NVML_ERROR_ALREADY_INITIALIZED       = 5
+NVML_ERROR_NOT_FOUND                 = 6
+NVML_ERROR_INSUFFICIENT_SIZE         = 7
+NVML_ERROR_INSUFFICIENT_POWER        = 8
+NVML_ERROR_DRIVER_NOT_LOADED         = 9
+NVML_ERROR_TIMEOUT                   = 10
+NVML_ERROR_IRQ_ISSUE                 = 11
+NVML_ERROR_LIBRARY_NOT_FOUND         = 12
+NVML_ERROR_FUNCTION_NOT_FOUND        = 13
+NVML_ERROR_CORRUPTED_INFOROM         = 14
+NVML_ERROR_GPU_IS_LOST               = 15
+NVML_ERROR_RESET_REQUIRED            = 16
+NVML_ERROR_OPERATING_SYSTEM          = 17
+NVML_ERROR_LIB_RM_VERSION_MISMATCH   = 18
+NVML_ERROR_IN_USE                    = 19
+NVML_ERROR_MEMORY                    = 20
+NVML_ERROR_NO_DATA                   = 21
+NVML_ERROR_VGPU_ECC_NOT_SUPPORTED    = 22
+NVML_ERROR_INSUFFICIENT_RESOURCES    = 23
+NVML_ERROR_FREQ_NOT_SUPPORTED        = 24
+NVML_ERROR_ARGUMENT_VERSION_MISMATCH = 25
+NVML_ERROR_DEPRECATED                = 26
+NVML_ERROR_NOT_READY                 = 27
+NVML_ERROR_GPU_NOT_FOUND             = 28
+NVML_ERROR_INVALID_STATE             = 29
+NVML_ERROR_UNKNOWN                   = 999
+
+_nvmlFanState_t = c_uint
+NVML_FAN_NORMAL             = 0
+NVML_FAN_FAILED             = 1
+
+_nvmlFanControlPolicy_t = c_uint
+NVML_FAN_POLICY_TEMPERATURE_CONTINOUS_SW = 0
+NVML_FAN_POLICY_MANUAL                   = 1
+
+_nvmlLedColor_t = c_uint
+NVML_LED_COLOR_GREEN        = 0
+NVML_LED_COLOR_AMBER        = 1
+
+_nvmlGpuOperationMode_t = c_uint
+NVML_GOM_ALL_ON                 = 0
+NVML_GOM_COMPUTE                = 1
+NVML_GOM_LOW_DP                 = 2
+
+_nvmlPageRetirementCause_t = c_uint
+NVML_PAGE_RETIREMENT_CAUSE_MULTIPLE_SINGLE_BIT_ECC_ERRORS = 0
+NVML_PAGE_RETIREMENT_CAUSE_DOUBLE_BIT_ECC_ERROR           = 1
+NVML_PAGE_RETIREMENT_CAUSE_COUNT                          = 2
+
+_nvmlRestrictedAPI_t = c_uint
+NVML_RESTRICTED_API_SET_APPLICATION_CLOCKS                = 0
+NVML_RESTRICTED_API_SET_AUTO_BOOSTED_CLOCKS               = 1
+NVML_RESTRICTED_API_COUNT                                 = 2
+
+_nvmlBridgeChipType_t = c_uint
+NVML_BRIDGE_CHIP_PLX = 0
+NVML_BRIDGE_CHIP_BRO4 = 1
+NVML_MAX_PHYSICAL_BRIDGE = 128
+
+_nvmlValueType_t = c_uint
+NVML_VALUE_TYPE_DOUBLE = 0
+NVML_VALUE_TYPE_UNSIGNED_INT = 1
+NVML_VALUE_TYPE_UNSIGNED_LONG = 2
+NVML_VALUE_TYPE_UNSIGNED_LONG_LONG = 3
+NVML_VALUE_TYPE_SIGNED_LONG_LONG = 4
+NVML_VALUE_TYPE_SIGNED_INT = 5
+NVML_VALUE_TYPE_UNSIGNED_SHORT = 6
+NVML_VALUE_TYPE_COUNT = 7
+
+_nvmlNvlinkVersion_t = c_uint
+NVML_NVLINK_VERSION_INVALID = 0
+NVML_NVLINK_VERSION_1_0 = 1
+NVML_NVLINK_VERSION_2_0 = 2
+NVML_NVLINK_VERSION_2_2 = 3
+NVML_NVLINK_VERSION_3_0 = 4
+NVML_NVLINK_VERSION_3_1 = 5
+NVML_NVLINK_VERSION_4_0 = 6
+NVML_NVLINK_VERSION_5_0 = 7
+
+_nvmlPerfPolicyType_t = c_uint
+NVML_PERF_POLICY_POWER = 0
+NVML_PERF_POLICY_THERMAL = 1
+NVML_PERF_POLICY_SYNC_BOOST = 2
+NVML_PERF_POLICY_BOARD_LIMIT = 3
+NVML_PERF_POLICY_LOW_UTILIZATION = 4
+NVML_PERF_POLICY_RELIABILITY = 5
+NVML_PERF_POLICY_TOTAL_APP_CLOCKS = 10
+NVML_PERF_POLICY_TOTAL_BASE_CLOCKS = 11
+NVML_PERF_POLICY_COUNT = 12
+
+_nvmlEncoderQueryType_t = c_uint
+NVML_ENCODER_QUERY_H264 = 0
+NVML_ENCODER_QUERY_HEVC = 1
+NVML_ENCODER_QUERY_AV1 = 2
+NVML_ENCODER_QUERY_UNKNOWN = 255
+
+_nvmlFBCSessionType_t = c_uint
+NVML_FBC_SESSION_TYPE_UNKNOWN = 0
+NVML_FBC_SESSION_TYPE_TOSYS = 1
+NVML_FBC_SESSION_TYPE_CUDA = 2
+NVML_FBC_SESSION_TYPE_VID = 3
+NVML_FBC_SESSION_TYPE_HWENC = 4
+
+_nvmlDetachGpuState_t = c_uint
+NVML_DETACH_GPU_KEEP = 0
+NVML_DETACH_GPU_REMOVE = 1
+
+_nvmlPcieLinkState_t = c_uint
+NVML_PCIE_LINK_KEEP = 0
+NVML_PCIE_LINK_SHUT_DOWN = 1
+
+_nvmlSamplingType_t = c_uint
+NVML_TOTAL_POWER_SAMPLES = 0
+NVML_GPU_UTILIZATION_SAMPLES = 1
+NVML_MEMORY_UTILIZATION_SAMPLES = 2
+NVML_ENC_UTILIZATION_SAMPLES = 3
+NVML_DEC_UTILIZATION_SAMPLES = 4
+NVML_PROCESSOR_CLK_SAMPLES = 5
+NVML_MEMORY_CLK_SAMPLES = 6
+NVML_MODULE_POWER_SAMPLES = 7
+NVML_JPG_UTILIZATION_SAMPLES = 8
+NVML_OFA_UTILIZATION_SAMPLES = 9
+NVML_SAMPLINGTYPE_COUNT = 10
+
+_nvmlPcieUtilCounter_t = c_uint
+NVML_PCIE_UTIL_TX_BYTES = 0
+NVML_PCIE_UTIL_RX_BYTES = 1
+NVML_PCIE_UTIL_COUNT = 2
+
+_nvmlGpuTopologyLevel_t = c_uint
+NVML_TOPOLOGY_INTERNAL = 0
+NVML_TOPOLOGY_SINGLE = 10
+NVML_TOPOLOGY_MULTIPLE = 20
+NVML_TOPOLOGY_HOSTBRIDGE = 30
+NVML_TOPOLOGY_NODE = 40
+NVML_TOPOLOGY_CPU = NVML_TOPOLOGY_NODE
+NVML_TOPOLOGY_SYSTEM = 50
+
+_nvmlGpuP2PCapsIndex_t = c_uint
+NVML_P2P_CAPS_INDEX_READ = 0,
+NVML_P2P_CAPS_INDEX_WRITE = 1
+NVML_P2P_CAPS_INDEX_NVLINK =2
+NVML_P2P_CAPS_INDEX_ATOMICS = 3
+#
+# NVML_P2P_CAPS_INDEX_PROP is deprecated.
+# Use NVML_P2P_CAPS_INDEX_PCI instead.
+#
+NVML_P2P_CAPS_INDEX_PROP = 4
+NVML_P2P_CAPS_INDEX_PCI = 4
+NVML_P2P_CAPS_INDEX_UNKNOWN = 5
+
+_nvmlGpuP2PStatus_t = c_uint
+NVML_P2P_STATUS_OK     = 0
+NVML_P2P_STATUS_CHIPSET_NOT_SUPPORED = 1
+NVML_P2P_STATUS_CHIPSET_NOT_SUPPORTED = NVML_P2P_STATUS_CHIPSET_NOT_SUPPORED
+NVML_P2P_STATUS_GPU_NOT_SUPPORTED = 2
+NVML_P2P_STATUS_IOH_TOPOLOGY_NOT_SUPPORTED =3
+NVML_P2P_STATUS_DISABLED_BY_REGKEY =4
+NVML_P2P_STATUS_NOT_SUPPORTED =5
+NVML_P2P_STATUS_UNKNOWN =6
+
+_nvmlDeviceArchitecture_t = c_uint
+NVML_DEVICE_ARCH_KEPLER   = 2
+NVML_DEVICE_ARCH_MAXWELL  = 3
+NVML_DEVICE_ARCH_PASCAL   = 4
+NVML_DEVICE_ARCH_VOLTA    = 5
+NVML_DEVICE_ARCH_TURING   = 6
+NVML_DEVICE_ARCH_AMPERE   = 7
+NVML_DEVICE_ARCH_ADA      = 8
+NVML_DEVICE_ARCH_HOPPER   = 9
+NVML_DEVICE_ARCH_BLACKWELL   = 10
+NVML_DEVICE_ARCH_T23X     = 11
+NVML_DEVICE_ARCH_UNKNOWN  = 0xffffffff
+
+# PCI bus Types
+_nvmlBusType_t = c_uint
+NVML_BUS_TYPE_UNKNOWN = 0
+NVML_BUS_TYPE_PCI     = 1
+NVML_BUS_TYPE_PCIE    = 2
+NVML_BUS_TYPE_FPCI    = 3
+NVML_BUS_TYPE_AGP     = 4
+
+_nvmlPowerSource_t = c_uint
+NVML_POWER_SOURCE_AC         = 0x00000000
+NVML_POWER_SOURCE_BATTERY    = 0x00000001
+NVML_POWER_SOURCE_UNDERSIZED = 0x00000002
+
+_nvmlAdaptiveClockInfoStatus_t = c_uint
+NVML_ADAPTIVE_CLOCKING_INFO_STATUS_DISABLED = 0x00000000
+NVML_ADAPTIVE_CLOCKING_INFO_STATUS_ENABLED = 0x00000001
+
+_nvmlClockLimitId_t = c_uint
+NVML_CLOCK_LIMIT_ID_RANGE_START = 0xffffff00
+NVML_CLOCK_LIMIT_ID_TDP         = 0xffffff01
+NVML_CLOCK_LIMIT_ID_UNLIMITED   = 0xffffff02
+
+_nvmlPcieLinkMaxSpeed_t = c_uint
+NVML_PCIE_LINK_MAX_SPEED_INVALID   = 0x00000000
+NVML_PCIE_LINK_MAX_SPEED_2500MBPS  = 0x00000001
+NVML_PCIE_LINK_MAX_SPEED_5000MBPS  = 0x00000002
+NVML_PCIE_LINK_MAX_SPEED_8000MBPS  = 0x00000003
+NVML_PCIE_LINK_MAX_SPEED_16000MBPS = 0x00000004
+NVML_PCIE_LINK_MAX_SPEED_32000MBPS = 0x00000005
+NVML_PCIE_LINK_MAX_SPEED_64000MBPS = 0x00000006
+
+_nvmlPcieAtomicsCapability_t = c_uint
+NVML_PCIE_ATOMICS_CAP_FETCHADD32  = 0x01
+NVML_PCIE_ATOMICS_CAP_FETCHADD64  = 0x02
+NVML_PCIE_ATOMICS_CAP_SWAP32      = 0x04
+NVML_PCIE_ATOMICS_CAP_SWAP64      = 0x08
+NVML_PCIE_ATOMICS_CAP_CAS32       = 0x10
+NVML_PCIE_ATOMICS_CAP_CAS64       = 0x20
+NVML_PCIE_ATOMICS_CAP_CAS128      = 0x40
+NVML_PCIE_ATOMICS_OPS_MAX         = 7
+
+_nvmlAffinityScope_t = c_uint
+NVML_AFFINITY_SCOPE_NODE   = 0
+NVML_AFFINITY_SCOPE_SOCKET = 1
+
+_nvmlDeviceGpuRecoveryAction_t = c_uint
+NVML_GPU_RECOVERY_ACTION_NONE        = 0
+NVML_GPU_RECOVERY_ACTION_GPU_RESET   = 1
+NVML_GPU_RECOVERY_ACTION_NODE_REBOOT = 2
+NVML_GPU_RECOVERY_ACTION_DRAIN_P2P   = 3
+NVML_GPU_RECOVERY_ACTION_DRAIN_AND_RESET = 4
+
+# C preprocessor defined values
+nvmlFlagDefault             = 0
+nvmlFlagForce               = 1
+NVML_INIT_FLAG_NO_GPUS      = 1
+NVML_INIT_FLAG_NO_ATTACH    = 2
+
+NVML_MAX_GPC_COUNT          = 32
+
+# buffer size
+NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE      = 16
+NVML_DEVICE_UUID_BUFFER_SIZE                 = 80
+NVML_DEVICE_UUID_V2_BUFFER_SIZE              = 96
+NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE       = 80
+NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE         = 80
+NVML_DEVICE_NAME_BUFFER_SIZE                 = 64
+NVML_DEVICE_NAME_V2_BUFFER_SIZE              = 96
+NVML_DEVICE_SERIAL_BUFFER_SIZE               = 30
+NVML_DEVICE_PART_NUMBER_BUFFER_SIZE          = 80
+NVML_DEVICE_GPU_PART_NUMBER_BUFFER_SIZE      = 80
+NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE        = 32
+NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE           = 32
+NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE        = 16
+NVML_GRID_LICENSE_BUFFER_SIZE                = 128
+NVML_VGPU_NAME_BUFFER_SIZE                   = 64
+NVML_GRID_LICENSE_FEATURE_MAX_COUNT          = 3
+NVML_VGPU_METADATA_OPAQUE_DATA_SIZE          = sizeof(c_uint) + 256
+NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE     = 256
+NVML_DEVICE_GPU_FRU_PART_NUMBER_BUFFER_SIZE  = 0x14 # NV2080_GPU_MAX_PRODUCT_PART_NUMBER_LENGTH
+NVML_PERF_MODES_BUFFER_SIZE                  = 2048
+
+# Format strings
+NVML_DEVICE_PCI_BUS_ID_LEGACY_FMT   = "%04X:%02X:%02X.0"
+NVML_DEVICE_PCI_BUS_ID_FMT          = "%08X:%02X:%02X.0"
+
+NVML_VALUE_NOT_AVAILABLE_ulonglong = c_ulonglong(-1)
+NVML_VALUE_NOT_AVAILABLE_uint = c_uint(-1)
+
+'''
+ Field Identifiers.
+
+ All Identifiers pertain to a device. Each ID is only used once and is guaranteed never to change.
+'''
+NVML_FI_DEV_ECC_CURRENT          = 1   # Current ECC mode. 1=Active. 0=Inactive
+NVML_FI_DEV_ECC_PENDING          = 2   # Pending ECC mode. 1=Active. 0=Inactive
+
+#ECC Count Totals
+NVML_FI_DEV_ECC_SBE_VOL_TOTAL    = 3   # Total single bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_TOTAL    = 4   # Total double bit volatile ECC errors
+NVML_FI_DEV_ECC_SBE_AGG_TOTAL    = 5   # Total single bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_DBE_AGG_TOTAL    = 6   # Total double bit aggregate (persistent) ECC errors
+#Individual ECC locations
+NVML_FI_DEV_ECC_SBE_VOL_L1       = 7   # L1 cache single bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_L1       = 8   # L1 cache double bit volatile ECC errors
+NVML_FI_DEV_ECC_SBE_VOL_L2       = 9   # L2 cache single bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_L2       = 10  # L2 cache double bit volatile ECC errors
+NVML_FI_DEV_ECC_SBE_VOL_DEV      = 11  # Device memory single bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_DEV      = 12  # Device memory double bit volatile ECC errors
+NVML_FI_DEV_ECC_SBE_VOL_REG      = 13  # Register file single bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_REG      = 14  # Register file double bit volatile ECC errors
+NVML_FI_DEV_ECC_SBE_VOL_TEX      = 15  # Texture memory single bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_TEX      = 16  # Texture memory double bit volatile ECC errors
+NVML_FI_DEV_ECC_DBE_VOL_CBU      = 17  # CBU double bit volatile ECC errors
+NVML_FI_DEV_ECC_SBE_AGG_L1       = 18  # L1 cache single bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_DBE_AGG_L1       = 19  # L1 cache double bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_SBE_AGG_L2       = 20  # L2 cache single bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_DBE_AGG_L2       = 21  # L2 cache double bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_SBE_AGG_DEV      = 22  # Device memory single bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_DBE_AGG_DEV      = 23  # Device memory double bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_SBE_AGG_REG      = 24  # Register File single bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_DBE_AGG_REG      = 25  # Register File double bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_SBE_AGG_TEX      = 26  # Texture memory single bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_DBE_AGG_TEX      = 27  # Texture memory double bit aggregate (persistent) ECC errors
+NVML_FI_DEV_ECC_DBE_AGG_CBU      = 28  # CBU double bit aggregate ECC errors
+
+# Page Retirement
+NVML_FI_DEV_RETIRED_SBE          = 29  # Number of retired pages because of single bit errors
+NVML_FI_DEV_RETIRED_DBE          = 30  # Number of retired pages because of double bit errors
+NVML_FI_DEV_RETIRED_PENDING      = 31  # If any pages are pending retirement. 1=yes. 0=no.
+
+# NvLink Flit Error Counters
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L0   = 32 # NVLink flow control CRC  Error Counter for Lane 0
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L1   = 33 # NVLink flow control CRC  Error Counter for Lane 1
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L2   = 34 # NVLink flow control CRC  Error Counter for Lane 2
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L3   = 35 # NVLink flow control CRC  Error Counter for Lane 3
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L4   = 36 # NVLink flow control CRC  Error Counter for Lane 4
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L5   = 37 # NVLink flow control CRC  Error Counter for Lane 5
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL = 38 # NVLink flow control CRC  Error Counter total for all Lanes
+
+# NvLink CRC Data Error Counters
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L0   = 39 # NVLink data CRC Error Counter for Lane 0
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L1   = 40 # NVLink data CRC Error Counter for Lane 1
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L2   = 41 # NVLink data CRC Error Counter for Lane 2
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L3   = 42 # NVLink data CRC Error Counter for Lane 3
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L4   = 43 # NVLink data CRC Error Counter for Lane 4
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L5   = 44 # NVLink data CRC Error Counter for Lane 5
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL = 45 # NvLink data CRC Error Counter total for all Lanes
+
+# NvLink Replay Error Counters
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L0     = 46 # NVLink Replay Error Counter for Lane 0
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L1     = 47 # NVLink Replay Error Counter for Lane 1
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L2     = 48 # NVLink Replay Error Counter for Lane 2
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L3     = 49 # NVLink Replay Error Counter for Lane 3
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L4     = 50 # NVLink Replay Error Counter for Lane 4
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L5     = 51 # NVLink Replay Error Counter for Lane 5
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL  = 52 # NVLink Replay Error Counter total for all Lanes
+
+# NvLink Recovery Error Counters
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L0   = 53 # NVLink Recovery Error Counter for Lane 0
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L1   = 54 # NVLink Recovery Error Counter for Lane 1
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L2   = 55 # NVLink Recovery Error Counter for Lane 2
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L3   = 56 # NVLink Recovery Error Counter for Lane 3
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L4   = 57 # NVLink Recovery Error Counter for Lane 4
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L5   = 58 # NVLink Recovery Error Counter for Lane 5
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL = 59 # NVLink Recovery Error Counter total for all Lanes
+
+# NvLink Bandwidth Counters
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L0    = 60 # NVLink Bandwidth Counter for Counter Set 0, Lane 0
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L1    = 61 # NVLink Bandwidth Counter for Counter Set 0, Lane 1
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L2    = 62 # NVLink Bandwidth Counter for Counter Set 0, Lane 2
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L3    = 63 # NVLink Bandwidth Counter for Counter Set 0, Lane 3
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L4    = 64 # NVLink Bandwidth Counter for Counter Set 0, Lane 4
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L5    = 65 # NVLink Bandwidth Counter for Counter Set 0, Lane 5
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_TOTAL = 66 # NVLink Bandwidth Counter Total for Counter Set 0, All Lanes
+
+# NvLink Bandwidth Counters
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L0    = 67 # NVLink Bandwidth Counter for Counter Set 1, Lane 0
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L1    = 68 # NVLink Bandwidth Counter for Counter Set 1, Lane 1
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L2    = 69 # NVLink Bandwidth Counter for Counter Set 1, Lane 2
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L3    = 70 # NVLink Bandwidth Counter for Counter Set 1, Lane 3
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L4    = 71 # NVLink Bandwidth Counter for Counter Set 1, Lane 4
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L5    = 72 # NVLink Bandwidth Counter for Counter Set 1, Lane 5
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_TOTAL = 73 # NVLink Bandwidth Counter Total for Counter Set 1, All Lanes
+
+# Perf Policy Counters
+NVML_FI_DEV_PERF_POLICY_POWER             = 74   # Perf Policy Counter for Power Policy
+NVML_FI_DEV_PERF_POLICY_THERMAL           = 75   # Perf Policy Counter for Thermal Policy
+NVML_FI_DEV_PERF_POLICY_SYNC_BOOST        = 76   # Perf Policy Counter for Sync boost Policy
+NVML_FI_DEV_PERF_POLICY_BOARD_LIMIT       = 77   # Perf Policy Counter for Board Limit
+NVML_FI_DEV_PERF_POLICY_LOW_UTILIZATION   = 78   # Perf Policy Counter for Low GPU Utilization Policy
+NVML_FI_DEV_PERF_POLICY_RELIABILITY       = 79   # Perf Policy Counter for Reliability Policy
+NVML_FI_DEV_PERF_POLICY_TOTAL_APP_CLOCKS  = 80   # Perf Policy Counter for Total App Clock Policy
+NVML_FI_DEV_PERF_POLICY_TOTAL_BASE_CLOCKS = 81   # Perf Policy Counter for Total Base Clocks Policy
+
+# Memory temperatures
+NVML_FI_DEV_MEMORY_TEMP  = 82 # Memory temperature for the device
+
+# Energy Counter
+NVML_FI_DEV_TOTAL_ENERGY_CONSUMPTION = 83 # Total energy consumption for the GPU in mJ since the driver was last reloaded
+
+# NVLink Speed
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L0     = 84
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L1     = 85
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L2     = 86
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L3     = 87
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L4     = 88
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L5     = 89
+NVML_FI_DEV_NVLINK_SPEED_MBPS_COMMON = 90
+
+# NVLink Link Count
+NVML_FI_DEV_NVLINK_LINK_COUNT = 91
+
+# Page Retirement pending fields
+NVML_FI_DEV_RETIRED_PENDING_SBE = 92
+NVML_FI_DEV_RETIRED_PENDING_DBE = 93
+
+# PCIe replay and replay rollover counters
+NVML_FI_DEV_PCIE_REPLAY_COUNTER = 94
+NVML_FI_DEV_PCIE_REPLAY_ROLLOVER_COUNTER = 95
+
+# NvLink Flit Error Counters
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L6   = 96 # NVLink flow control CRC  Error Counter for Lane 6
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L7   = 97 # NVLink flow control CRC  Error Counter for Lane 7
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L8   = 98 # NVLink flow control CRC  Error Counter for Lane 8
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L9   = 99 # NVLink flow control CRC  Error Counter for Lane 9
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L10  = 100 # NVLink flow control CRC  Error Counter for Lane 10
+NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L11  = 101 # NVLink flow control CRC  Error Counter for Lane 11
+
+# NvLink CRC Data Error Counters
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L6   = 102 # NVLink data CRC Error Counter for Lane 6
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L7   = 103 # NVLink data CRC Error Counter for Lane 7
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L8   = 104 # NVLink data CRC Error Counter for Lane 8
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L9   = 105 # NVLink data CRC Error Counter for Lane 9
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L10  = 106 # NVLink data CRC Error Counter for Lane 10
+NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L11  = 107 # NVLink data CRC Error Counter for Lane 11
+
+# NvLink Replay Error Counters
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L6     = 108 # NVLink Replay Error Counter for Lane 6
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L7     = 109 # NVLink Replay Error Counter for Lane 7
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L8     = 110 # NVLink Replay Error Counter for Lane 8
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L9     = 111 # NVLink Replay Error Counter for Lane 9
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L10    = 112 # NVLink Replay Error Counter for Lane 10
+NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L11    = 113 # NVLink Replay Error Counter for Lane 11
+
+# NvLink Recovery Error Counters
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L6   = 114 # NVLink Recovery Error Counter for Lane 6
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L7   = 115 # NVLink Recovery Error Counter for Lane 7
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L8   = 116 # NVLink Recovery Error Counter for Lane 8
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L9   = 117 # NVLink Recovery Error Counter for Lane 9
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L10  = 118 # NVLink Recovery Error Counter for Lane 10
+NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L11  = 119 # NVLink Recovery Error Counter for Lane 11
+
+# NvLink Bandwidth Counters
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L6    = 120 # NVLink Bandwidth Counter for Counter Set 0, Lane 6
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L7    = 121 # NVLink Bandwidth Counter for Counter Set 0, Lane 7
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L8    = 122 # NVLink Bandwidth Counter for Counter Set 0, Lane 8
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L9    = 123 # NVLink Bandwidth Counter for Counter Set 0, Lane 9
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L10   = 124 # NVLink Bandwidth Counter for Counter Set 0, Lane 10
+NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L11   = 125 # NVLink Bandwidth Counter for Counter Set 0, Lane 11
+
+# NvLink Bandwidth Counters
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L6    = 126 # NVLink Bandwidth Counter for Counter Set 1, Lane 6
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L7    = 127 # NVLink Bandwidth Counter for Counter Set 1, Lane 7
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L8    = 128 # NVLink Bandwidth Counter for Counter Set 1, Lane 8
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L9    = 129 # NVLink Bandwidth Counter for Counter Set 1, Lane 9
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L10   = 130 # NVLink Bandwidth Counter for Counter Set 1, Lane 10
+NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L11   = 131 # NVLink Bandwidth Counter for Counter Set 1, Lane 11
+
+# NVLink Speed
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L6     = 132
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L7     = 133
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L8     = 134
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L9     = 135
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L10    = 136
+NVML_FI_DEV_NVLINK_SPEED_MBPS_L11    = 137
+
+# NVLink Throughput Counters
+NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_TX = 138 # NVLink TX Data throughput in KiB
+NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_RX = 139 # NVLink RX Data throughput in KiB
+NVML_FI_DEV_NVLINK_THROUGHPUT_RAW_TX  = 140 # NVLink TX Data + protocol overhead in KiB
+NVML_FI_DEV_NVLINK_THROUGHPUT_RAW_RX  = 141 # NVLink RX Data + protocol overhead in KiB
+
+# Row Remapper
+NVML_FI_DEV_REMAPPED_COR        = 142
+NVML_FI_DEV_REMAPPED_UNC        = 143
+NVML_FI_DEV_REMAPPED_PENDING    = 144
+NVML_FI_DEV_REMAPPED_FAILURE    = 145
+
+#Remote device NVLink ID
+NVML_FI_DEV_NVLINK_REMOTE_NVLINK_ID = 146
+
+# Number of NVLinks connected to NVSwitch
+NVML_FI_DEV_NVSWITCH_CONNECTED_LINK_COUNT = 147
+
+# NvLink ECC Data Error Counters
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L0    = 148 #< NVLink data ECC Error Counter for Link 0
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L1    = 149 #< NVLink data ECC Error Counter for Link 1
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L2    = 150 #< NVLink data ECC Error Counter for Link 2
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L3    = 151 #< NVLink data ECC Error Counter for Link 3
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L4    = 152 #< NVLink data ECC Error Counter for Link 4
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L5    = 153 #< NVLink data ECC Error Counter for Link 5
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L6    = 154 #< NVLink data ECC Error Counter for Link 6
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L7    = 155 #< NVLink data ECC Error Counter for Link 7
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L8    = 156 #< NVLink data ECC Error Counter for Link 8
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L9    = 157 #< NVLink data ECC Error Counter for Link 9
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L10   = 158 #< NVLink data ECC Error Counter for Link 10
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L11   = 159 #< NVLink data ECC Error Counter for Link 11
+NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_TOTAL = 160 #< NvLink data ECC Error Counter total for all Links
+
+NVML_FI_DEV_NVLINK_ERROR_DL_REPLAY            = 161
+NVML_FI_DEV_NVLINK_ERROR_DL_RECOVERY          = 162
+NVML_FI_DEV_NVLINK_ERROR_DL_CRC               = 163
+NVML_FI_DEV_NVLINK_GET_SPEED                  = 164
+NVML_FI_DEV_NVLINK_GET_STATE                  = 165
+NVML_FI_DEV_NVLINK_GET_VERSION                = 166
+
+NVML_FI_DEV_NVLINK_GET_POWER_STATE            = 167
+NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD        = 168
+
+NVML_FI_DEV_PCIE_L0_TO_RECOVERY_COUNTER       = 169
+
+NVML_FI_DEV_C2C_LINK_COUNT                    = 170
+NVML_FI_DEV_C2C_LINK_GET_STATUS               = 171
+NVML_FI_DEV_C2C_LINK_GET_MAX_BW               = 172
+
+NVML_FI_DEV_PCIE_COUNT_CORRECTABLE_ERRORS     = 173
+NVML_FI_DEV_PCIE_COUNT_NAKS_RECEIVED          = 174
+NVML_FI_DEV_PCIE_COUNT_RECEIVER_ERROR         = 175
+NVML_FI_DEV_PCIE_COUNT_BAD_TLP                = 176
+NVML_FI_DEV_PCIE_COUNT_NAKS_SENT              = 177
+NVML_FI_DEV_PCIE_COUNT_BAD_DLLP               = 178
+NVML_FI_DEV_PCIE_COUNT_NON_FATAL_ERROR        = 179
+NVML_FI_DEV_PCIE_COUNT_FATAL_ERROR            = 180
+NVML_FI_DEV_PCIE_COUNT_UNSUPPORTED_REQ        = 181
+NVML_FI_DEV_PCIE_COUNT_LCRC_ERROR             = 182
+NVML_FI_DEV_PCIE_COUNT_LANE_ERROR             = 183
+
+NVML_FI_DEV_IS_RESETLESS_MIG_SUPPORTED        = 184
+
+NVML_FI_DEV_POWER_AVERAGE                     = 185
+NVML_FI_DEV_POWER_INSTANT                     = 186
+NVML_FI_DEV_POWER_MIN_LIMIT                   = 187
+NVML_FI_DEV_POWER_MAX_LIMIT                   = 188
+NVML_FI_DEV_POWER_DEFAULT_LIMIT               = 189
+NVML_FI_DEV_POWER_CURRENT_LIMIT               = 190
+NVML_FI_DEV_ENERGY                            = 191
+NVML_FI_DEV_POWER_REQUESTED_LIMIT             = 192
+
+NVML_FI_DEV_TEMPERATURE_SHUTDOWN_TLIMIT       = 193
+NVML_FI_DEV_TEMPERATURE_SLOWDOWN_TLIMIT       = 194
+NVML_FI_DEV_TEMPERATURE_MEM_MAX_TLIMIT        = 195
+NVML_FI_DEV_TEMPERATURE_GPU_MAX_TLIMIT        = 196
+
+NVML_FI_DEV_PCIE_COUNT_TX_BYTES               = 197
+NVML_FI_DEV_PCIE_COUNT_RX_BYTES               = 198
+
+NVML_FI_DEV_IS_MIG_MODE_INDEPENDENT_MIG_QUERY_CAPABLE   = 199
+
+NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_MAX              = 200
+
+NVML_FI_DEV_NVLINK_COUNT_XMIT_PACKETS                    = 201
+NVML_FI_DEV_NVLINK_COUNT_XMIT_BYTES                      = 202
+NVML_FI_DEV_NVLINK_COUNT_RCV_PACKETS                     = 203
+NVML_FI_DEV_NVLINK_COUNT_RCV_BYTES                       = 204
+NVML_FI_DEV_NVLINK_COUNT_VL15_DROPPED                    = 205 # Deprecated, do not use
+NVML_FI_DEV_NVLINK_COUNT_MALFORMED_PACKET_ERRORS         = 206
+NVML_FI_DEV_NVLINK_COUNT_BUFFER_OVERRUN_ERRORS           = 207
+NVML_FI_DEV_NVLINK_COUNT_RCV_ERRORS                      = 208
+NVML_FI_DEV_NVLINK_COUNT_RCV_REMOTE_ERRORS               = 209
+NVML_FI_DEV_NVLINK_COUNT_RCV_GENERAL_ERRORS              = 210
+NVML_FI_DEV_NVLINK_COUNT_LOCAL_LINK_INTEGRITY_ERRORS     = 211
+NVML_FI_DEV_NVLINK_COUNT_XMIT_DISCARDS                   = 212
+
+NVML_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_SUCCESSFUL_EVENTS = 213
+NVML_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_FAILED_EVENTS     = 214
+NVML_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_EVENTS            = 215
+
+NVML_FI_DEV_NVLINK_COUNT_RAW_BER_LANE0                   = 216  # Deprecated, do not use
+NVML_FI_DEV_NVLINK_COUNT_RAW_BER_LANE1                   = 217  # Deprecated, do not use
+NVML_FI_DEV_NVLINK_COUNT_RAW_BER                         = 218  # Deprecated, do not use
+NVML_FI_DEV_NVLINK_COUNT_EFFECTIVE_ERRORS                = 219
+NVML_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER                   = 220
+NVML_FI_DEV_NVLINK_COUNT_SYMBOL_ERRORS                   = 221
+NVML_FI_DEV_NVLINK_COUNT_SYMBOL_BER                      = 222
+
+NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_MIN               = 223
+NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_UNITS             = 224 # Values are in the form NVML_NVLINK_LOW_POWER_THRESHOLD_UNIT_*
+NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_SUPPORTED         = 225
+
+NVML_FI_DEV_RESET_STATUS                                 = 226 # Deprecated use NVML_FI_DEV_GET_GPU_RECOVERY_ACTION instead 
+NVML_FI_DEV_DRAIN_AND_RESET_STATUS                       = 227 # Deprecated use NVML_FI_DEV_GET_GPU_RECOVERY_ACTION instead
+NVML_FI_DEV_PCIE_OUTBOUND_ATOMICS_MASK                   = 228
+NVML_FI_DEV_PCIE_INBOUND_ATOMICS_MASK                    = 229
+NVML_FI_DEV_GET_GPU_RECOVERY_ACTION                      = 230
+
+NVML_FI_DEV_C2C_LINK_ERROR_INTR                          = 231
+NVML_FI_DEV_C2C_LINK_ERROR_REPLAY                        = 232
+NVML_FI_DEV_C2C_LINK_ERROR_REPLAY_B2B                    = 233
+NVML_FI_DEV_C2C_LINK_POWER_STATE                         = 234
+
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_0                   = 235
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_1                   = 236
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_2                   = 237
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_3                   = 238
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_4                   = 239
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_5                   = 240
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_6                   = 241
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_7                   = 242
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_8                   = 243
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_9                   = 244
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_10                  = 245
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_11                  = 246
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_12                  = 247
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_13                  = 248
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_14                  = 249
+NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_15                  = 250
+NVML_FI_DEV_CLOCKS_EVENT_REASON_SW_POWER_CAP             = NVML_FI_DEV_PERF_POLICY_POWER
+NVML_FI_DEV_CLOCKS_EVENT_REASON_SYNC_BOOST               = NVML_FI_DEV_PERF_POLICY_SYNC_BOOST
+NVML_FI_DEV_CLOCKS_EVENT_REASON_SW_THERM_SLOWDOWN        = 251
+NVML_FI_DEV_CLOCKS_EVENT_REASON_HW_THERM_SLOWDOWN        = 252
+NVML_FI_DEV_CLOCKS_EVENT_REASON_HW_POWER_BRAKE_SLOWDOWN  = 253
+NVML_FI_DEV_POWER_SYNC_BALANCING_FREQ                    = 254
+NVML_FI_DEV_POWER_SYNC_BALANCING_AF                      = 255
+NVML_FI_PWR_SMOOTHING_ENABLED                                   = 256 # Enablement (0/DISABLED or 1/ENABLED)
+NVML_FI_PWR_SMOOTHING_PRIV_LVL                                  = 257 # Current privilege level
+NVML_FI_PWR_SMOOTHING_IMM_RAMP_DOWN_ENABLED                     = 258 # Immediate ramp down enablement (0/DISABLED or 1/ENABLED)
+NVML_FI_PWR_SMOOTHING_APPLIED_TMP_CEIL                          = 259 # Applied TMP ceiling value
+NVML_FI_PWR_SMOOTHING_APPLIED_TMP_FLOOR                         = 260 # Applied TMP floor value
+NVML_FI_PWR_SMOOTHING_MAX_PERCENT_TMP_FLOOR_SETTING             = 261 # Max % TMP Floor value
+NVML_FI_PWR_SMOOTHING_MIN_PERCENT_TMP_FLOOR_SETTING             = 262 # Min % TMP Floor value
+NVML_FI_PWR_SMOOTHING_HW_CIRCUITRY_PERCENT_LIFETIME_REMAINING   = 263 # HW Circuitry % lifetime remaining
+NVML_FI_PWR_SMOOTHING_MAX_NUM_PRESET_PROFILES                   = 264 # Max number of preset profiles
+NVML_FI_PWR_SMOOTHING_PROFILE_PERCENT_TMP_FLOOR                 = 265 # % TMP floor for a given profile
+NVML_FI_PWR_SMOOTHING_PROFILE_RAMP_UP_RATE                      = 266 # Ramp up rate in mW/s for a given profile
+NVML_FI_PWR_SMOOTHING_PROFILE_RAMP_DOWN_RATE                    = 267 # Ramp down rate in mW/s for a given profile
+NVML_FI_PWR_SMOOTHING_PROFILE_RAMP_DOWN_HYST_VAL                = 268 # Ramp down hysteresis value in ms for a given profile
+NVML_FI_PWR_SMOOTHING_ACTIVE_PRESET_PROFILE                     = 269 # Active preset profile number
+NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_PERCENT_TMP_FLOOR          = 270 # % TMP floor for a given profile
+NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_UP_RATE               = 271 # Ramp up rate in mW/s for a given profile
+NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_RATE             = 272 # Ramp down rate in mW/s for a given profile
+NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_HYST_VAL         = 273 # Ramp down hysteresis value in ms for a given profile
+NVML_FI_MAX = 274 # One greater than the largest field ID defined above
+
+# NVML_FI_DEV_NVLINK_GET_STATE state enums
+NVML_NVLINK_STATE_INACTIVE = 0x0
+NVML_NVLINK_STATE_ACTIVE   = 0x1
+NVML_NVLINK_STATE_SLEEP    = 0x2
+
+NVML_NVLINK_LOW_POWER_THRESHOLD_UNIT_100US = 0 # NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_UNITS
+NVML_NVLINK_LOW_POWER_THRESHOLD_UNIT_50US  = 1 # NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_UNITS
+
+# NVML_FI_DEV_C2C_LINK_POWER_STATE state enums
+NVML_C2C_POWER_STATE_FULL_POWER = 0
+NVML_C2C_POWER_STATE_LOW_POWER = 1
+
+## Enums needed for the method nvmlDeviceGetVirtualizationMode and nvmlDeviceSetVirtualizationMode
+NVML_GPU_VIRTUALIZATION_MODE_NONE        = 0  # Represents Bare Metal GPU
+NVML_GPU_VIRTUALIZATION_MODE_PASSTHROUGH = 1  # Device is associated with GPU-Passthorugh
+NVML_GPU_VIRTUALIZATION_MODE_VGPU        = 2  # Device is associated with vGPU inside virtual machine.
+NVML_GPU_VIRTUALIZATION_MODE_HOST_VGPU   = 3  # Device is associated with VGX hypervisor in vGPU mode
+NVML_GPU_VIRTUALIZATION_MODE_HOST_VSGA   = 4  # Device is associated with VGX hypervisor in vSGA mode
+
+## Lib loading ##
+nvmlLib = None
+libLoadLock = threading.Lock()
+_nvmlLib_refcount = 0 # Incremented on each nvmlInit and decremented on nvmlShutdown
+
+## vGPU Management
+_nvmlVgpuTypeId_t   = c_uint
+_nvmlVgpuInstance_t = c_uint
+
+_nvmlVgpuVmIdType_t = c_uint
+NVML_VGPU_VM_ID_DOMAIN_ID    = 0
+NVML_VGPU_VM_ID_UUID         = 1
+
+_nvmlGridLicenseFeatureCode_t = c_uint
+NVML_GRID_LICENSE_FEATURE_CODE_UNKNOWN      = 0
+NVML_GRID_LICENSE_FEATURE_CODE_VGPU         = 1
+NVML_GRID_LICENSE_FEATURE_CODE_NVIDIA_RTX   = 2
+NVML_GRID_LICENSE_FEATURE_CODE_VWORKSTATION = 2 # deprecated, use NVML_GRID_LICENSE_FEATURE_CODE_NVIDIA_RTX.
+NVML_GRID_LICENSE_FEATURE_CODE_GAMING       = 3
+NVML_GRID_LICENSE_FEATURE_CODE_COMPUTE      = 4
+
+_nvmlGridLicenseExpiryStatus_t = c_uint8
+NVML_GRID_LICENSE_EXPIRY_NOT_AVAILABLE    = 0,   # Expiry information not available
+NVML_GRID_LICENSE_EXPIRY_INVALID          = 1,   # Invalid expiry or error fetching expiry
+NVML_GRID_LICENSE_EXPIRY_VALID            = 2,   # Valid expiry
+NVML_GRID_LICENSE_EXPIRY_NOT_APPLICABLE   = 3,   # Expiry not applicable
+NVML_GRID_LICENSE_EXPIRY_PERMANENT        = 4,   # Permanent expiry
+
+_nvmlVgpuCapability_t = c_uint
+NVML_VGPU_CAP_NVLINK_P2P                    = 0  # vGPU P2P over NVLink is supported
+NVML_VGPU_CAP_GPUDIRECT                     = 1  # GPUDirect capability is supported
+NVML_VGPU_CAP_MULTI_VGPU_EXCLUSIVE          = 2  # vGPU profile cannot be mixed with other vGPU profiles in same VM
+NVML_VGPU_CAP_EXCLUSIVE_TYPE                = 3  # vGPU profile cannot run on a GPU alongside other profiles of different type
+NVML_VGPU_CAP_EXCLUSIVE_SIZE                = 4  # vGPU profile cannot run on a GPU alongside other profiles of different size
+NVML_VGPU_CAP_COUNT                         = 5
+
+_nvmlVgpuDriverCapability_t = c_uint
+NVML_VGPU_DRIVER_CAP_HETEROGENEOUS_MULTI_VGPU   = 0  # Supports mixing of different vGPU profiles within one guest VM
+NVML_VGPU_DRIVER_CAP_WARM_UPDATE                = 1  # Supports FSR and warm update of vGPU host driver without terminating the running guest VM
+NVML_VGPU_DRIVER_CAP_COUNT                      = 2
+
+_nvmlDeviceVgpuCapability_t = c_uint
+NVML_DEVICE_VGPU_CAP_FRACTIONAL_MULTI_VGPU             = 0  # Query whether the fractional vGPU profiles on this GPU can be used in multi-vGPU configurations
+NVML_DEVICE_VGPU_CAP_HETEROGENEOUS_TIMESLICE_PROFILES  = 1  # Query whether the GPU supports concurrent execution of timesliced vGPU profiles of differing types
+NVML_DEVICE_VGPU_CAP_HETEROGENEOUS_TIMESLICE_SIZES     = 2  # Query whether the GPU supports concurrent execution of timesliced vGPU profiles of differing framebuffer sizes
+NVML_DEVICE_VGPU_CAP_READ_DEVICE_BUFFER_BW             = 3  # Query the GPU's read_device_buffer expected bandwidth capacity in megabytes per second
+NVML_DEVICE_VGPU_CAP_WRITE_DEVICE_BUFFER_BW            = 4  # Query the GPU's write_device_buffer expected bandwidth capacity in megabytes per second
+NVML_DEVICE_VGPU_CAP_DEVICE_STREAMING                  = 5  # Query whether the vGPU profiles on the GPU supports migration data streaming
+NVML_DEVICE_VGPU_CAP_MINI_QUARTER_GPU                  = 6  # Set/Get support of mini-quarter vGPU profiles
+NVML_DEVICE_VGPU_CAP_COMPUTE_MEDIA_ENGINE_GPU          = 7  # Set/Get support for compute media engine vGPU profiles
+NVML_DEVICE_VGPU_CAP_WARM_UPDATE                       = 8  # Query whether the GPU supports FSR and warm update
+NVML_DEVICE_VGPU_CAP_HOMOGENEOUS_PLACEMENTS            = 9  # Query whether the GPU supports reporting of placements of timesliced vGPU profiles with identical framebuffer sizes
+NVML_DEVICE_VGPU_CAP_MIG_TIMESLICING_SUPPORTED         = 10 # Query whether the GPU supports timesliced vGPU on MIG
+NVML_DEVICE_VGPU_CAP_MIG_TIMESLICING_ENABLED           = 11 # Set/Get MIG timesliced mode reporting, without impacting the underlying functionality
+NVML_DEVICE_VGPU_CAP_COUNT                             = 12
+
+_nvmlVgpuGuestInfoState_t = c_uint
+NVML_VGPU_INSTANCE_GUEST_INFO_STATE_UNINITIALIZED = 0
+NVML_VGPU_INSTANCE_GUEST_INFO_STATE_INITIALIZED   = 1
+
+_nvmlVgpuVmCompatibility_t = c_uint
+NVML_VGPU_VM_COMPATIBILITY_NONE         = 0x0
+NVML_VGPU_VM_COMPATIBILITY_COLD         = 0x1
+NVML_VGPU_VM_COMPATIBILITY_HIBERNATE    = 0x2
+NVML_VGPU_VM_COMPATIBILITY_SLEEP        = 0x4
+NVML_VGPU_VM_COMPATIBILITY_LIVE         = 0x8
+
+_nvmlVgpuPgpuCompatibilityLimitCode_t = c_uint
+NVML_VGPU_COMPATIBILITY_LIMIT_NONE          = 0x0
+NVML_VGPU_COMPATIBILITY_LIMIT_HOST_DRIVER   = 0x1
+NVML_VGPU_COMPATIBILITY_LIMIT_GUEST_DRIVER  = 0x2
+NVML_VGPU_COMPATIBILITY_LIMIT_GPU           = 0x4
+NVML_VGPU_COMPATIBILITY_LIMIT_OTHER         = 0x80000000
+
+_nvmlHostVgpuMode_t = c_uint
+NVML_HOST_VGPU_MODE_NON_SRIOV   = 0
+NVML_HOST_VGPU_MODE_SRIOV       = 1
+
+_nvmlConfComputeGpusReadyState_t = c_uint
+NVML_CC_ACCEPTING_CLIENT_REQUESTS_FALSE = 0
+NVML_CC_ACCEPTING_CLIENT_REQUESTS_TRUE = 1
+
+_nvmlConfComputeGpuCaps_t = c_uint
+NVML_CC_SYSTEM_GPUS_CC_NOT_CAPABLE = 0
+NVML_CC_SYSTEM_GPUS_CC_CAPABLE = 1
+
+_nvmlConfComputeCpuCaps_t = c_uint
+NVML_CC_SYSTEM_CPU_CAPS_NONE = 0
+NVML_CC_SYSTEM_CPU_CAPS_AMD_SEV = 1
+NVML_CC_SYSTEM_CPU_CAPS_INTEL_TDX = 2
+NVML_CC_SYSTEM_CPU_CAPS_AMD_SEV_SNP = 3
+NVML_CC_SYSTEM_CPU_CAPS_AMD_SNP_VTOM = 4
+
+_nvmlConfComputeDevToolsMode_t = c_uint
+NVML_CC_SYSTEM_DEVTOOLS_MODE_OFF = 0
+NVML_CC_SYSTEM_DEVTOOLS_MODE_ON = 1
+
+NVML_CC_SYSTEM_MULTIGPU_NONE = 0
+NVML_CC_SYSTEM_MULTIGPU_PROTECTED_PCIE = 1
+ 
+NVML_CC_SYSTEM_ENVIRONMENT_UNAVAILABLE = 0
+NVML_CC_SYSTEM_ENVIRONMENT_SIM = 1
+NVML_CC_SYSTEM_ENVIRONMENT_PROD = 2
+ 
+_nvmlConfComputeCcFeature_t = c_uint
+NVML_CC_SYSTEM_FEATURE_DISABLED = 0
+NVML_CC_SYSTEM_FEATURE_ENABLED = 1
+
+_nvmlConfComputeCcKeyRotationThreshAttackerAdv_t = c_uint
+NVML_CC_KEY_ROTATION_THRESH_ATTACKER_ADVANTAGE_MIN = 50
+NVML_CC_KEY_ROTATION_THRESH_ATTACKER_ADVANTAGE_MAX = 65
+
+# GSP firmware
+NVML_GSP_FIRMWARE_VERSION_BUF_SIZE = 0x40
+
+class NVMLLibraryMismatchError(Exception):
+    pass
+
+## Error Checking ##
+class NVMLError(Exception):
+    _valClassMapping = dict()
+    # List of currently known error codes
+    _errcode_to_string = {
+        NVML_ERROR_UNINITIALIZED:       "Uninitialized",
+        NVML_ERROR_INVALID_ARGUMENT:    "Invalid Argument",
+        NVML_ERROR_NOT_SUPPORTED:       "Not Supported",
+        NVML_ERROR_NO_PERMISSION:       "Insufficient Permissions",
+        NVML_ERROR_ALREADY_INITIALIZED: "Already Initialized",
+        NVML_ERROR_NOT_FOUND:           "Not Found",
+        NVML_ERROR_INSUFFICIENT_SIZE:   "Insufficient Size",
+        NVML_ERROR_INSUFFICIENT_POWER:  "Insufficient External Power",
+        NVML_ERROR_DRIVER_NOT_LOADED:   "Driver Not Loaded",
+        NVML_ERROR_TIMEOUT:             "Timeout",
+        NVML_ERROR_IRQ_ISSUE:           "Interrupt Request Issue",
+        NVML_ERROR_LIBRARY_NOT_FOUND:   "NVML Shared Library Not Found",
+        NVML_ERROR_FUNCTION_NOT_FOUND:  "Function Not Found",
+        NVML_ERROR_CORRUPTED_INFOROM:   "Corrupted infoROM",
+        NVML_ERROR_GPU_IS_LOST:         "GPU is lost",
+        NVML_ERROR_RESET_REQUIRED:      "GPU requires restart",
+        NVML_ERROR_OPERATING_SYSTEM:    "The operating system has blocked the request.",
+        NVML_ERROR_LIB_RM_VERSION_MISMATCH: "RM has detected an NVML/RM version mismatch.",
+        NVML_ERROR_MEMORY:              "Insufficient Memory",
+        NVML_ERROR_UNKNOWN:             "Unknown Error",
+        }
+    def __new__(typ, value):
+        '''
+        Maps value to a proper subclass of NVMLError.
+        See _extractNVMLErrorsAsClasses function for more details
+        '''
+        if typ == NVMLError:
+            typ = NVMLError._valClassMapping.get(value, typ)
+        obj = Exception.__new__(typ)
+        obj.value = value
+        return obj
+    def __str__(self):
+        try:
+            if self.value not in NVMLError._errcode_to_string:
+                NVMLError._errcode_to_string[self.value] = str(nvmlErrorString(self.value))
+            return NVMLError._errcode_to_string[self.value]
+        except NVMLError:
+            return "NVML Error with code %d" % self.value
+    def __eq__(self, other):
+        return self.value == other.value
+
+def nvmlExceptionClass(nvmlErrorCode):
+    if nvmlErrorCode not in NVMLError._valClassMapping:
+        raise ValueError('nvmlErrorCode %s is not valid' % nvmlErrorCode)
+    return NVMLError._valClassMapping[nvmlErrorCode]
+
+def _extractNVMLErrorsAsClasses():
+    '''
+    Generates a hierarchy of classes on top of NVMLError class.
+
+    Each NVML Error gets a new NVMLError subclass. This way try,except blocks can filter appropriate
+    exceptions more easily.
+
+    NVMLError is a parent class. Each NVML_ERROR_* gets it's own subclass.
+    e.g. NVML_ERROR_ALREADY_INITIALIZED will be turned into NVMLError_AlreadyInitialized
+    '''
+    this_module = sys.modules[__name__]
+    nvmlErrorsNames = [x for x in dir(this_module) if x.startswith("NVML_ERROR_")]
+    for err_name in nvmlErrorsNames:
+        # e.g. Turn NVML_ERROR_ALREADY_INITIALIZED into NVMLError_AlreadyInitialized
+        class_name = "NVMLError_" + string.capwords(err_name.replace("NVML_ERROR_", ""), "_").replace("_", "")
+        err_val = getattr(this_module, err_name)
+        def gen_new(val):
+            def new(typ):
+                obj = NVMLError.__new__(typ, val)
+                return obj
+            return new
+        new_error_class = type(class_name, (NVMLError,), {'__new__': gen_new(err_val)})
+        new_error_class.__module__ = __name__
+        setattr(this_module, class_name, new_error_class)
+        NVMLError._valClassMapping[err_val] = new_error_class
+_extractNVMLErrorsAsClasses()
+
+def _nvmlCheckReturn(ret):
+    if (ret != NVML_SUCCESS):
+        raise NVMLError(ret)
+    return ret
+
+## Function access ##
+_nvmlGetFunctionPointer_cache = dict() # function pointers are cached to prevent unnecessary libLoadLock locking
+def _nvmlGetFunctionPointer(name):
+    global nvmlLib
+
+    if name in _nvmlGetFunctionPointer_cache:
+        return _nvmlGetFunctionPointer_cache[name]
+
+    libLoadLock.acquire()
+    try:
+        # ensure library was loaded
+        if (nvmlLib == None):
+            raise NVMLError(NVML_ERROR_UNINITIALIZED)
+        try:
+            _nvmlGetFunctionPointer_cache[name] = getattr(nvmlLib, name)
+            return _nvmlGetFunctionPointer_cache[name]
+        except AttributeError:
+            raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND)
+    finally:
+        # lock is always freed
+        libLoadLock.release()
+
+## Alternative object
+# Allows the object to be printed
+# Allows mismatched types to be assigned
+#  - like None when the Structure variant requires c_uint
+class nvmlFriendlyObject(object):
+    def __init__(self, dictionary):
+        for x in dictionary:
+            setattr(self, x, dictionary[x])
+    def __str__(self):
+        return self.__dict__.__str__()
+
+def nvmlStructToFriendlyObject(struct):
+    d = {}
+    for x in struct._fields_:
+        key = x[0]
+        value = getattr(struct, key)
+        # only need to convert from bytes if bytes, no need to check python version.
+        d[key] = value.decode() if isinstance(value, bytes) else value
+    obj = nvmlFriendlyObject(d)
+    return obj
+
+# pack the object so it can be passed to the NVML library
+def nvmlFriendlyObjectToStruct(obj, model):
+    for x in model._fields_:
+        key = x[0]
+        value = obj.__dict__[key]
+        # any c_char_p in python3 needs to be bytes, default encoding works fine.
+        if sys.version_info >= (3,):
+            setattr(model, key, value.encode())
+        else:
+            setattr(model, key, value)
+    return model
+
+## Unit structures
+class struct_c_nvmlUnit_t(Structure):
+    pass # opaque handle
+c_nvmlUnit_t = POINTER(struct_c_nvmlUnit_t)
+
+class _PrintableStructure(Structure):
+    """
+    Abstract class that produces nicer __str__ output than ctypes.Structure.
+    e.g. instead of:
+      >>> print str(obj)
+      
+    this class will print
+      class_name(field_name: formatted_value, field_name: formatted_value)
+
+    _fmt_ dictionary of  -> 
+    e.g. class that has _field_ 'hex_value', c_uint could be formatted with
+      _fmt_ = {"hex_value" : "%08X"}
+    to produce nicer output.
+    Default fomratting string for all fields can be set with key "" like:
+      _fmt_ = {"" : "%d MHz"} # e.g all values are numbers in MHz.
+    If not set it's assumed to be just "%s"
+
+    Exact format of returned str from this class is subject to change in the future.
+    """
+    _fmt_ = {}
+    def __str__(self):
+        result = []
+        for x in self._fields_:
+            key = x[0]
+            value = getattr(self, key)
+            fmt = "%s"
+            if key in self._fmt_:
+                fmt = self._fmt_[key]
+            elif "" in self._fmt_:
+                fmt = self._fmt_[""]
+            result.append(("%s: " + fmt) % (key, value))
+        return self.__class__.__name__ + "(" +  ", ".join(result) + ")"
+
+    def __getattribute__(self, name):
+        res = super(_PrintableStructure, self).__getattribute__(name)
+        # need to convert bytes to unicode for python3 don't need to for python2
+        # Python 2 strings are of both str and bytes
+        # Python 3 strings are not of type bytes
+        # ctypes should convert everything to the correct values otherwise
+        if isinstance(res, bytes):
+            if isinstance(res, str):
+                return res
+            return res.decode()
+        return res
+
+    def __setattr__(self, name, value):
+        if isinstance(value, str):
+            # encoding a python2 string returns the same value, since python2 strings are bytes already
+            # bytes passed in python3 will be ignored.
+            value = value.encode()
+        super(_PrintableStructure, self).__setattr__(name, value)
+
+class c_nvmlUnitInfo_t(_PrintableStructure):
+    _fields_ = [
+        ('name', c_char * 96),
+        ('id', c_char * 96),
+        ('serial', c_char * 96),
+        ('firmwareVersion', c_char * 96),
+    ]
+
+class c_nvmlC2cModeInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('isC2cEnabled', c_uint)
+    ]
+
+nvmlC2cModeInfo_v1 = 0x1000008;
+
+class c_nvmlLedState_t(_PrintableStructure):
+    _fields_ = [
+        ('cause', c_char * 256),
+        ('color', _nvmlLedColor_t),
+    ]
+
+class c_nvmlPSUInfo_t(_PrintableStructure):
+    _fields_ = [
+        ('state', c_char * 256),
+        ('current', c_uint),
+        ('voltage', c_uint),
+        ('power', c_uint),
+    ]
+
+class c_nvmlUnitFanInfo_t(_PrintableStructure):
+    _fields_ = [
+        ('speed', c_uint),
+        ('state', _nvmlFanState_t),
+    ]
+
+class c_nvmlUnitFanSpeeds_t(_PrintableStructure):
+    _fields_ = [
+        ('fans', c_nvmlUnitFanInfo_t * 24),
+        ('count', c_uint)
+    ]
+
+## Device structures
+class struct_c_nvmlDevice_t(Structure):
+    pass # opaque handle
+c_nvmlDevice_t = POINTER(struct_c_nvmlDevice_t)
+
+class nvmlPciInfoExt_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('domain', c_uint),
+        ('bus', c_uint),
+        ('device', c_uint),
+        ('pciDeviceId', c_uint),
+        ('pciSubSystemId', c_uint),
+        ('baseClass', c_uint),
+        ('subClass', c_uint),
+        ('busId', c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE),
+    ]
+    _fmt_ = {
+            'version'        : "0x%04X",
+            'domain'         : "0x%04X",
+            'bus'            : "0x%02X",
+            'device'         : "0x%02X",
+            'pciDeviceId'    : "0x%08X",
+            'pciSubSystemId' : "0x%08X",
+            'baseClass'      : "0x%01X",
+            'subClass'       : "0x%01X",
+            }
+
+nvmlPciInfoExt_v1 = 0x1000040
+
+# Legacy pciInfo used for _v1 and _v2
+class nvmlPciInfo_v2_t(_PrintableStructure):
+    _fields_ = [
+        ('busId', c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE),
+        ('domain', c_uint),
+        ('bus', c_uint),
+        ('device', c_uint),
+        ('pciDeviceId', c_uint),
+
+        # Added in 2.285
+        ('pciSubSystemId', c_uint),
+        ('reserved0', c_uint),
+        ('reserved1', c_uint),
+        ('reserved2', c_uint),
+        ('reserved3', c_uint),
+    ]
+    _fmt_ = {
+            'domain'         : "0x%04X",
+            'bus'            : "0x%02X",
+            'device'         : "0x%02X",
+            'pciDeviceId'    : "0x%08X",
+            'pciSubSystemId' : "0x%08X",
+            }
+
+class nvmlPciInfo_t(_PrintableStructure):
+    _fields_ = [
+        # Moved to the new busId location below
+        ('busIdLegacy', c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE),
+        ('domain', c_uint),
+        ('bus', c_uint),
+        ('device', c_uint),
+        ('pciDeviceId', c_uint),
+
+        # Added in 2.285
+        ('pciSubSystemId', c_uint),
+        # New busId replaced the long deprecated and reserved fields with a
+        # field of the same size in 9.0
+        ('busId', c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE),
+    ]
+    _fmt_ = {
+            'domain'         : "0x%08X",
+            'bus'            : "0x%02X",
+            'device'         : "0x%02X",
+            'pciDeviceId'    : "0x%08X",
+            'pciSubSystemId' : "0x%08X",
+            }
+
+class c_nvmlSystemDriverBranchInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ("branch", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE),
+    ]
+
+SystemDriverBranchInfo_v1 = 0x1000054
+
+class c_nvmlExcludedDeviceInfo_t(_PrintableStructure):
+    _fields_ = [
+        ('pci', nvmlPciInfo_t),
+        ('uuid', c_char * NVML_DEVICE_UUID_BUFFER_SIZE)
+    ]
+
+class nvmlNvLinkUtilizationControl_t(_PrintableStructure):
+    _fields_ = [
+        ('units', _nvmlNvLinkUtilizationCountUnits_t),
+        ('pktfilter', _nvmlNvLinkUtilizationCountPktTypes_t),
+    ]
+
+class c_nvmlMemory_t(_PrintableStructure):
+    _fields_ = [
+        ('total', c_ulonglong),
+        ('free', c_ulonglong),
+        ('used', c_ulonglong),
+    ]
+    _fmt_ = {'': "%d B"}
+
+class c_nvmlMemory_v2_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('total', c_ulonglong),
+        ('reserved', c_ulonglong),
+        ('free', c_ulonglong),
+        ('used', c_ulonglong),
+    ]
+    _fmt_ = {'': "%d B"}
+
+nvmlMemory_v2 = 0x02000028
+
+class c_nvmlBAR1Memory_t(_PrintableStructure):
+    _fields_ = [
+        ('bar1Total', c_ulonglong),
+        ('bar1Free', c_ulonglong),
+        ('bar1Used', c_ulonglong),
+    ]
+    _fmt_ = {'': "%d B"}
+
+class nvmlClkMonFaultInfo_t(Structure):
+    _fields_ = [("clkApiDomain", c_uint),
+                ("clkDomainFaultMask", c_uint)
+    ]
+
+MAX_CLK_DOMAINS = 32
+
+class nvmlClkMonStatus_t(Structure):
+    _fields_ = [("bGlobalStatus", c_uint),
+                ("clkMonListSize", c_uint),
+                ("clkMonList", nvmlClkMonFaultInfo_t * MAX_CLK_DOMAINS)
+    ]
+
+# On Windows with the WDDM driver, usedGpuMemory is reported as None
+# Code that processes this structure should check for None, I.E.
+#
+# if (info.usedGpuMemory == None):
+#     # TODO handle the error
+#     pass
+# else:
+#    print("Using %d MiB of memory" % (info.usedGpuMemory / 1024 / 1024))
+# endif
+#
+# See NVML documentation for more information
+class c_nvmlProcessInfo_v2_t(_PrintableStructure):
+    _fields_ = [
+        ('pid', c_uint),
+        ('usedGpuMemory', c_ulonglong),
+        ('gpuInstanceId', c_uint),
+        ('computeInstanceId', c_uint),
+    ]
+    _fmt_ = {'usedGpuMemory': "%d B"}
+
+c_nvmlProcessInfo_v3_t = c_nvmlProcessInfo_v2_t
+
+c_nvmlProcessInfo_t = c_nvmlProcessInfo_v3_t
+
+_nvmlProcessMode_t = c_uint
+NVML_PROCESS_MODE_COMPUTE  = 0
+NVML_PROCESS_MODE_GRAPHICS = 1
+NVML_PROCESS_MODE_MPS      = 2
+
+class c_nvmlProcessDetail_v1_t(Structure):
+    _fields_ = [
+        ('pid', c_uint),
+        ('usedGpuMemory', c_ulonglong),
+        ('gpuInstanceId', c_uint),
+        ('computeInstanceId', c_uint),
+        ('usedGpuCcProtectedMemory', c_ulonglong),
+    ]
+
+class c_nvmlProcessDetailList_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('mode', _nvmlProcessMode_t),
+        ('numProcArrayEntries', c_uint),
+        ('procArray', POINTER(c_nvmlProcessDetail_v1_t)),
+    ]
+    _fmt_ = {'numProcArrayEntries': "%d B"}
+
+c_nvmlProcessDetailList_t = c_nvmlProcessDetailList_v1_t
+
+nvmlProcessDetailList_v1 = 0x1000018
+
+class c_nvmlBridgeChipInfo_t(_PrintableStructure):
+    _fields_ = [
+        ('type', _nvmlBridgeChipType_t),
+        ('fwVersion', c_uint),
+    ]
+
+class c_nvmlBridgeChipHierarchy_t(_PrintableStructure):
+    _fields_ = [
+        ('bridgeCount', c_uint),
+        ('bridgeChipInfo', c_nvmlBridgeChipInfo_t * 128),
+    ]
+
+class c_nvmlEccErrorCounts_t(_PrintableStructure):
+    _fields_ = [
+        ('l1Cache', c_ulonglong),
+        ('l2Cache', c_ulonglong),
+        ('deviceMemory', c_ulonglong),
+        ('registerFile', c_ulonglong),
+    ]
+
+class c_nvmlUtilization_t(_PrintableStructure):
+    _fields_ = [
+        ('gpu', c_uint),
+        ('memory', c_uint),
+    ]
+    _fmt_ = {'': "%d %%"}
+
+# Added in 2.285
+class c_nvmlHwbcEntry_t(_PrintableStructure):
+    _fields_ = [
+        ('hwbcId', c_uint),
+        ('firmwareVersion', c_char * 32),
+    ]
+
+class c_nvmlValue_t(Union):
+    _fields_ = [
+        ('dVal', c_double),
+        ('uiVal', c_uint),
+        ('ulVal', c_ulong),
+        ('ullVal', c_ulonglong),
+        ('sllVal', c_longlong),
+        ('siVal', c_int),
+        ('usVal', c_ushort),
+    ]
+
+class c_nvmlSample_t(_PrintableStructure):
+    _fields_ = [
+        ('timeStamp', c_ulonglong),
+        ('sampleValue', c_nvmlValue_t),
+    ]
+
+class c_nvmlViolationTime_t(_PrintableStructure):
+    _fields_ = [
+        ('referenceTime', c_ulonglong),
+        ('violationTime', c_ulonglong),
+    ]
+
+class c_nvmlFieldValue_t(_PrintableStructure):
+    _fields_ = [
+        ('fieldId', c_uint32),
+        ('scopeId', c_uint32),
+        ('timestamp', c_int64),
+        ('latencyUsec', c_int64),
+        ('valueType', _nvmlValueType_t),
+        ('nvmlReturn', _nvmlReturn_t),
+        ('value', c_nvmlValue_t)
+    ]
+
+NVML_NVLINK_TOTAL_SUPPORTED_BW_MODES = 23
+
+nvmlNvlinkSupportedBwModes_v1 = 0x100001c
+class c_nvmlNvlinkSupportedBwModes_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('bwModes', c_uint8 * NVML_NVLINK_TOTAL_SUPPORTED_BW_MODES),
+        ('totalBwModes', c_uint8)
+    ]
+
+    def __init__(self):
+        super(c_nvmlNvlinkSupportedBwModes_v1_t, self).__init__(version=nvmlNvlinkSupportedBwModes_v1)
+
+nvmlNvlinkGetBwMode_v1 = 0x100000c
+class c_nvmlNvlinkGetBwMode_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('bIsBest', c_uint),
+        ('bwMode', c_uint8)
+    ]
+
+    def __init__(self):
+        super(c_nvmlNvlinkGetBwMode_v1_t, self).__init__(version=nvmlNvlinkGetBwMode_v1)
+
+nvmlNvlinkSetBwMode_v1 = 0x100000c
+class c_nvmlNvlinkSetBwMode_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('bSetBest', c_uint),
+        ('bwMode', c_uint8)
+    ]
+
+    def __init__(self):
+        super(c_nvmlNvlinkSetBwMode_v1_t, self).__init__(version=nvmlNvlinkSetBwMode_v1)
+
+class c_nvmlVgpuHeterogeneousMode_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('mode', c_uint),
+    ]
+
+VgpuHeterogeneousMode_v1 = 0x1000008
+
+class c_nvmlVgpuPlacementId_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('placementId', c_uint),
+    ]
+
+VgpuPlacementId_v1 = 0x1000008
+
+class c_nvmlVgpuPlacementList_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('count', c_uint),
+        ('placementSize', c_uint),
+        ('placementIds', POINTER(c_uint)),
+    ]
+
+VgpuPlacementList_v1 = 0x1000018
+
+NVML_VGPU_PGPU_HETEROGENEOUS_MODE   = 0
+NVML_VGPU_PGPU_HOMOGENEOUS_MODE     = 1
+
+class c_nvmlVgpuPlacementList_v2_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('placementSize', c_uint),
+        ('count', c_uint),
+        ('placementIds', POINTER(c_uint)),
+        ('mode', c_uint),
+    ]
+
+VgpuPlacementList_v2 = 0x2000020
+
+class c_nvmlVgpuTypeBar1Info_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('bar1Size', c_ulonglong),
+    ]
+
+VgpuTypeBar1Info_v1 = 0x1000010
+
+class c_nvmlVgpuInstanceUtilizationSample_t(_PrintableStructure):
+    _fields_ = [
+        ('vgpuInstance', _nvmlVgpuInstance_t),
+        ('timeStamp', c_ulonglong),
+        ('smUtil', c_nvmlValue_t),
+        ('memUtil', c_nvmlValue_t),
+        ('encUtil', c_nvmlValue_t),
+        ('decUtil', c_nvmlValue_t),
+    ]
+
+class c_nvmlVgpuInstanceUtilizationInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('timeStamp', c_ulonglong),
+        ('vgpuInstance', _nvmlVgpuInstance_t),
+        ('smUtil', c_nvmlValue_t),
+        ('memUtil', c_nvmlValue_t),
+        ('encUtil', c_nvmlValue_t),
+        ('decUtil', c_nvmlValue_t),
+        ('jpgUtil', c_nvmlValue_t),
+        ('ofaUtil', c_nvmlValue_t),
+    ]
+
+class c_nvmlVgpuInstancesUtilizationInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('sampleValType', _nvmlValueType_t),
+        ('vgpuInstanceCount', c_uint),
+        ('lastSeenTimeStamp', c_ulonglong),
+        ('vgpuUtilArray', POINTER(c_nvmlVgpuInstanceUtilizationInfo_v1_t)),
+    ]
+
+VgpuInstancesUtilizationInfo_v1 = 0x01000020
+
+class c_nvmlVgpuProcessUtilizationSample_t(_PrintableStructure):
+    _fields_ = [
+        ('vgpuInstance', _nvmlVgpuInstance_t),
+        ('pid', c_uint),
+        ('processName', c_char * NVML_VGPU_NAME_BUFFER_SIZE),
+        ('timeStamp', c_ulonglong),
+        ('smUtil', c_uint),
+        ('memUtil', c_uint),
+        ('encUtil', c_uint),
+        ('decUtil', c_uint),
+    ]
+
+class c_nvmlVgpuProcessUtilizationInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('processName', c_char * NVML_VGPU_NAME_BUFFER_SIZE),
+        ('timeStamp', c_ulonglong),
+        ('vgpuInstance', _nvmlVgpuInstance_t),
+        ('pid', c_uint),
+        ('smUtil', c_uint),
+        ('memUtil', c_uint),
+        ('encUtil', c_uint),
+        ('decUtil', c_uint),
+        ('jpgUtil', c_uint),
+        ('ofaUtil', c_uint),
+    ]
+
+class c_nvmlVgpuProcessesUtilizationInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('vgpuProcessCount', c_uint),
+        ('lastSeenTimeStamp', c_ulonglong),
+        ('vgpuProcUtilArray', POINTER(c_nvmlVgpuProcessUtilizationInfo_v1_t)),
+    ]
+
+VgpuProcessesUtilizationInfo_v1 = 0x01000018
+
+class nvmlVgpuRuntimeState_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('size', c_ulonglong),
+    ]
+
+VgpuRuntimeState_v1 = 0x1000010
+
+class c_nvmlVgpuLicenseExpiry_t(_PrintableStructure):
+    _fields_ = [
+        ('year',    c_uint32),
+        ('month',   c_uint16),
+        ('day',     c_uint16),
+        ('hour',    c_uint16),
+        ('min',     c_uint16),
+        ('sec',     c_uint16),
+        ('status',  c_uint8),
+    ]
+
+NVML_GRID_LICENSE_STATE_UNKNOWN                 = 0
+NVML_GRID_LICENSE_STATE_UNINITIALIZED           = 1
+NVML_GRID_LICENSE_STATE_UNLICENSED_UNRESTRICTED = 2
+NVML_GRID_LICENSE_STATE_UNLICENSED_RESTRICTED   = 3
+NVML_GRID_LICENSE_STATE_UNLICENSED              = 4
+NVML_GRID_LICENSE_STATE_LICENSED                = 5
+
+class c_nvmlVgpuLicenseInfo_t(_PrintableStructure):
+    _fields_ = [
+        ('isLicensed',      c_uint8),
+        ('licenseExpiry',   c_nvmlVgpuLicenseExpiry_t),
+        ('currentState',    c_uint),
+    ]
+
+class c_nvmlEncoderSession_t(_PrintableStructure):
+    _fields_ = [
+        ('sessionId', c_uint),
+        ('pid', c_uint),
+        ('vgpuInstance', _nvmlVgpuInstance_t),
+        ('codecType', c_uint),
+        ('hResolution', c_uint),
+        ('vResolution', c_uint),
+        ('averageFps', c_uint),
+        ('encodeLatency', c_uint),
+    ]
+
+class c_nvmlProcessUtilizationSample_t(_PrintableStructure):
+    _fields_ = [
+        ('pid', c_uint),
+        ('timeStamp', c_ulonglong),
+        ('smUtil', c_uint),
+        ('memUtil', c_uint),
+        ('encUtil', c_uint),
+        ('decUtil', c_uint),
+    ]
+
+class c_nvmlProcessUtilizationInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('timeStamp', c_ulonglong),
+        ('pid', c_uint),
+        ('smUtil', c_uint),
+        ('memUtil', c_uint),
+        ('encUtil', c_uint),
+        ('decUtil', c_uint),
+        ('jpgUtil', c_uint),
+        ('ofaUtil', c_uint),
+    ]
+
+class c_nvmlProcessesUtilizationInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('processSamplesCount', c_uint),
+        ('lastSeenTimeStamp', c_ulonglong),
+        ('procUtilArray', POINTER(c_nvmlProcessUtilizationInfo_v1_t)),
+    ]
+
+ProcessesUtilizationInfo_v1 = 0x01000018
+
+class c_nvmlGridLicenseExpiry_t(_PrintableStructure):
+    _fields_ = [
+        ('year',    c_uint32),
+        ('month',   c_uint16),
+        ('day',     c_uint16),
+        ('hour',    c_uint16),
+        ('min',     c_uint16),
+        ('sec',     c_uint16),
+        ('status',  c_uint8),
+    ]
+
+class c_nvmlGridLicensableFeature_v4_t(_PrintableStructure):
+    _fields_ = [
+        ('featureCode',    _nvmlGridLicenseFeatureCode_t),
+        ('featureState',   c_uint),
+        ('licenseInfo',    c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+        ('productName',    c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+        ('featureEnabled', c_uint),
+        ('licenseExpiry',  c_nvmlGridLicenseExpiry_t),
+    ]
+
+class c_nvmlGridLicensableFeatures_v4_t(_PrintableStructure):
+    _fields_ = [
+        ('isGridLicenseSupported',  c_int),
+        ('licensableFeaturesCount', c_uint),
+        ('gridLicensableFeatures',  c_nvmlGridLicensableFeature_v4_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT),
+    ]
+
+class c_nvmlGridLicensableFeature_v3_t(_PrintableStructure):
+    _fields_ = [
+        ('featureCode', _nvmlGridLicenseFeatureCode_t),
+        ('featureState', c_uint),
+        ('licenseInfo', c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+        ('productName', c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+        ('featureEnabled', c_uint),
+    ]
+
+class c_nvmlGridLicensableFeatures_v3_t(_PrintableStructure):
+    _fields_ = [
+        ('isGridLicenseSupported', c_int),
+        ('licensableFeaturesCount', c_uint),
+        ('gridLicensableFeatures', c_nvmlGridLicensableFeature_v3_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT),
+    ]
+
+class c_nvmlGridLicensableFeature_v2_t(_PrintableStructure):
+    _fields_ = [
+        ('featureCode', _nvmlGridLicenseFeatureCode_t),
+        ('featureState', c_uint),
+        ('licenseInfo', c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+        ('productName', c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+    ]
+
+class c_nvmlGridLicensableFeatures_v2_t(_PrintableStructure):
+    _fields_ = [
+        ('isGridLicenseSupported', c_int),
+        ('licensableFeaturesCount', c_uint),
+        ('gridLicensableFeatures', c_nvmlGridLicensableFeature_v2_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT),
+    ]
+
+class c_nvmlGridLicensableFeature_t(_PrintableStructure):
+    _fields_ = [
+        ('featureCode', _nvmlGridLicenseFeatureCode_t),
+        ('featureState', c_uint),
+        ('licenseInfo', c_char * NVML_GRID_LICENSE_BUFFER_SIZE),
+    ]
+
+class c_nvmlGridLicensableFeatures_t(_PrintableStructure):
+    _fields_ = [
+        ('isGridLicenseSupported', c_int),
+        ('licensableFeaturesCount', c_uint),
+        ('gridLicensableFeatures', c_nvmlGridLicensableFeature_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT),
+    ]
+
+class c_nvmlMarginTemperature_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('marginTemperature', c_int),
+    ]
+
+nvmlMarginTemperature_v1 = 0x1000008
+
+NVML_DEVICE_UUID_ASCII_LEN  = 41
+NVML_DEVICE_UUID_BINARY_LEN = 16
+
+NVML_UUID_TYPE_NONE   = 0
+NVML_UUID_TYPE_ASCII  = 1
+NVML_UUID_TYPE_BINARY = 2
+
+class c_nvmlUUIDValue_t(Union):
+    _fields_ = [
+        ('str',   c_char * NVML_DEVICE_UUID_ASCII_LEN),
+        ('bytes', c_ubyte * NVML_DEVICE_UUID_BINARY_LEN),
+    ]
+
+nvmlUUID_v1 = 0x1000034
+class c_nvmlUUID_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('type', c_uint),
+        ('value', c_nvmlUUIDValue_t),
+    ]
+
+    def __init__(self):
+        super(c_nvmlUUID_t, self).__init__(version=nvmlUUID_v1)
+
+## Event structures
+class struct_c_nvmlEventSet_t(Structure):
+    pass # opaque handle
+c_nvmlEventSet_t = POINTER(struct_c_nvmlEventSet_t)
+
+nvmlEventTypeSingleBitEccError      = 0x0000000000000001
+nvmlEventTypeDoubleBitEccError      = 0x0000000000000002
+nvmlEventTypePState                 = 0x0000000000000004
+nvmlEventTypeXidCriticalError       = 0x0000000000000008
+nvmlEventTypeClock                  = 0x0000000000000010
+nvmlEventTypePowerSourceChange      = 0x0000000000000080
+nvmlEventMigConfigChange            = 0x0000000000000100
+nvmlEventTypeSingleBitEccErrorStorm = 0x0000000000000200
+nvmlEventTypeDramRetirementEvent    = 0x0000000000000400
+nvmlEventTypeDramRetirementFailure  = 0x0000000000000800
+nvmlEventTypeNonFatalPoisonError    = 0x0000000000001000
+nvmlEventTypeFatalPoisonError       = 0x0000000000002000
+nvmlEventTypeGpuUnavailableError    = 0x0000000000004000
+nvmlEventTypeGpuRecoveryAction      = 0x0000000000008000
+nvmlEventTypeNone                   = 0x0000000000000000
+nvmlEventTypeAll                    = (
+                                        nvmlEventTypeNone
+                                        | nvmlEventTypeSingleBitEccError
+                                        | nvmlEventTypeDoubleBitEccError
+                                        | nvmlEventTypePState
+                                        | nvmlEventTypeClock
+                                        | nvmlEventTypePowerSourceChange
+                                        | nvmlEventTypeXidCriticalError
+                                        | nvmlEventMigConfigChange
+                                        | nvmlEventTypeSingleBitEccErrorStorm
+                                        | nvmlEventTypeDramRetirementEvent
+                                        | nvmlEventTypeDramRetirementFailure
+                                        | nvmlEventTypeNonFatalPoisonError
+                                        | nvmlEventTypeFatalPoisonError
+                                        | nvmlEventTypeGpuUnavailableError
+                                        | nvmlEventTypeGpuRecoveryAction
+                                        )
+
+## Clock Event Reasons defines
+nvmlClocksEventReasonGpuIdle              = 0x0000000000000001
+nvmlClocksEventReasonApplicationsClocksSetting = 0x0000000000000002
+nvmlClocksEventReasonUserDefinedClocks         = nvmlClocksEventReasonApplicationsClocksSetting # deprecated, use nvmlClocksEventReasonApplicationsClocksSetting
+nvmlClocksEventReasonSwPowerCap           = 0x0000000000000004
+nvmlClocksEventReasonHwSlowdown           = 0x0000000000000008
+nvmlClocksEventReasonSyncBoost            = 0x0000000000000010
+nvmlClocksEventReasonSwThermalSlowdown    = 0x0000000000000020
+nvmlClocksEventReasonHwThermalSlowdown    = 0x0000000000000040
+nvmlClocksEventReasonHwPowerBrakeSlowdown = 0x0000000000000080
+nvmlClocksEventReasonDisplayClockSetting  = 0x0000000000000100
+nvmlClocksEventReasonNone                 = 0x0000000000000000
+nvmlClocksEventReasonAll                  = (
+                                                  nvmlClocksEventReasonNone |
+                                                  nvmlClocksEventReasonGpuIdle |
+                                                  nvmlClocksEventReasonApplicationsClocksSetting |
+                                                  nvmlClocksEventReasonSwPowerCap |
+                                                  nvmlClocksEventReasonHwSlowdown |
+                                                  nvmlClocksEventReasonSyncBoost |
+                                                  nvmlClocksEventReasonSwThermalSlowdown |
+                                                  nvmlClocksEventReasonHwThermalSlowdown |
+                                                  nvmlClocksEventReasonHwPowerBrakeSlowdown |
+                                                  nvmlClocksEventReasonDisplayClockSetting
+                                               )
+
+## Following have been deprecated
+nvmlClocksThrottleReasonGpuIdle              = 0x0000000000000001
+nvmlClocksThrottleReasonApplicationsClocksSetting = 0x0000000000000002
+nvmlClocksThrottleReasonUserDefinedClocks         = nvmlClocksThrottleReasonApplicationsClocksSetting # deprecated, use nvmlClocksThrottleReasonApplicationsClocksSetting
+nvmlClocksThrottleReasonSwPowerCap           = 0x0000000000000004
+nvmlClocksThrottleReasonHwSlowdown           = 0x0000000000000008
+nvmlClocksThrottleReasonSyncBoost            = 0x0000000000000010
+nvmlClocksThrottleReasonSwThermalSlowdown    = 0x0000000000000020
+nvmlClocksThrottleReasonHwThermalSlowdown    = 0x0000000000000040
+nvmlClocksThrottleReasonHwPowerBrakeSlowdown = 0x0000000000000080
+nvmlClocksThrottleReasonDisplayClockSetting  = 0x0000000000000100
+nvmlClocksThrottleReasonNone                 = 0x0000000000000000
+nvmlClocksThrottleReasonAll                  = (
+                                                  nvmlClocksThrottleReasonNone |
+                                                  nvmlClocksThrottleReasonGpuIdle |
+                                                  nvmlClocksThrottleReasonApplicationsClocksSetting |
+                                                  nvmlClocksThrottleReasonSwPowerCap |
+                                                  nvmlClocksThrottleReasonHwSlowdown |
+                                                  nvmlClocksThrottleReasonSyncBoost |
+                                                  nvmlClocksThrottleReasonSwThermalSlowdown |
+                                                  nvmlClocksThrottleReasonHwThermalSlowdown |
+                                                  nvmlClocksThrottleReasonHwPowerBrakeSlowdown |
+                                                  nvmlClocksThrottleReasonDisplayClockSetting
+                                               )
+
+class c_nvmlEventData_t(_PrintableStructure):
+    _fields_ = [
+        ('device', c_nvmlDevice_t),
+        ('eventType', c_ulonglong),
+        ('eventData', c_ulonglong),
+        ('gpuInstanceId', c_uint),
+        ('computeInstanceId', c_uint)
+    ]
+    _fmt_ = {'eventType': "0x%08X"}
+
+class struct_c_nvmlSystemEventSet_t(Structure):
+    pass # opaque handle
+c_nvmlSystemEventSet_t = POINTER(struct_c_nvmlSystemEventSet_t)
+
+nvmlSystemEventTypeGpuDriverUnbind = 0x0000000000000001
+nvmlSystemEventTypeGpuDriverBind   = 0x0000000000000002
+
+nvmlSystemEventTypeCount = 2
+
+nvmlSystemEventSetCreateRequest_v1 = 0x1000010
+class c_nvmlSystemEventSetCreateRequest_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('set', c_nvmlSystemEventSet_t)
+    ]
+
+    def __init__(self):
+        super(c_nvmlSystemEventSetCreateRequest_v1_t, self).__init__(version=nvmlSystemEventSetCreateRequest_v1)
+
+nvmlSystemEventSetFreeRequest_v1 = 0x1000010
+class c_nvmlSystemEventSetFreeRequest_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('set', c_nvmlSystemEventSet_t)
+    ]
+
+    def __init__(self):
+        super(c_nvmlSystemEventSetFreeRequest_v1_t, self).__init__(version=nvmlSystemEventSetFreeRequest_v1)
+
+nvmlSystemRegisterEventRequest_v1 = 0x1000018
+class c_nvmlSystemRegisterEventRequest_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('eventTypes', c_ulonglong),
+        ('set', c_nvmlSystemEventSet_t)
+    ]
+
+    def __init__(self):
+        super(c_nvmlSystemRegisterEventRequest_v1_t, self).__init__(version=nvmlSystemRegisterEventRequest_v1)
+
+class c_nvmlSystemEventData_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('eventType', c_ulonglong),
+        ('gpuId', c_uint)
+    ]
+    _fmt_ = {'eventType': "0x%08X"}
+
+nvmlSystemEventSetWaitRequest_v1 = 0x1000020
+class c_nvmlSystemEventSetWaitRequest_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),                            # input/output
+        ('timeoutms', c_uint),                          # input
+        ('set', c_nvmlSystemEventSet_t),                # input
+        ('data', POINTER(c_nvmlSystemEventData_v1_t)),  # input/output. Buffer owned by caller
+        ('dataSize', c_uint),                           # input
+        ('numEvent', c_uint)                            # output. Number of event recorded
+    ]
+
+    def __init__(self):
+        super(c_nvmlSystemEventSetWaitRequest_v1_t, self).__init__(version=nvmlSystemEventSetWaitRequest_v1)
+
+# C APIs for system events
+def c_nvmlSystemEventSetCreate(req):
+    """
+    C API binding for c_nvmlSystemEventSetFree
+    :param req: reference of c_nvmlSystemEventSetCreateRequest_v1_t
+    """
+    fn = _nvmlGetFunctionPointer("nvmlSystemEventSetCreate")
+    ret = fn(req)
+    _nvmlCheckReturn(ret)
+
+def c_nvmlSystemEventSetFree(req):
+    """
+    C API binding for c_nvmlSystemEventSetFree
+    :param req: reference of c_nvmlSystemEventSetFreeRequest_v1_t
+    """
+    fn = _nvmlGetFunctionPointer("nvmlSystemEventSetFree")
+    ret = fn(req)
+    _nvmlCheckReturn(ret)
+
+def c_nvmlSystemRegisterEvents(req):
+    """
+    C API binding for nvmlSystemRegisterEvents
+    :param req: reference of c_nvmlSystemRegisterEventRequest_v1_t
+    """
+    fn = _nvmlGetFunctionPointer("nvmlSystemRegisterEvents")
+    ret = fn(req)
+    _nvmlCheckReturn(ret)
+
+def c_nvmlSystemEventSetWait(req):
+    """
+    C API binding for nvmlSystemEventSetWait
+    :param req: reference of c_nvmlSystemEventSetWaitRequest_v1_t
+    """
+    fn = _nvmlGetFunctionPointer("nvmlSystemEventSetWait")
+    ret = fn(req)
+    # Accept TIMEOUT error
+    if ret == NVML_ERROR_TIMEOUT:
+        return ret
+    _nvmlCheckReturn(ret)
+
+# Pythonic APIs for system events
+# No Pythonic API compability support
+# When user upgrades to a new NVML Python binding,
+# user should upgrade their code against API change.
+def nvmlSystemEventSetCreate(version):
+    """
+    Create an NVML system event set.
+    :param version: The version of the request.
+    :return: A ctypes pointer to the created c_nvmlSystemEventSet_t structure.
+    """
+    if version != nvmlSystemEventSetCreateRequest_v1:
+        raise NVMLError(NVML_ERROR_ARGUMENT_VERSION_MISMATCH)
+
+    # Prepare the request structure
+    req = c_nvmlSystemEventSetCreateRequest_v1_t()
+    req.version = version
+    req.set = None  # Initialize as NULL
+
+    c_nvmlSystemEventSetCreate(byref(req))
+
+    # Return the created set handle
+    if not req.set:
+        raise NVMLError(NVML_ERROR_MEMORY)
+    return req.set
+
+def nvmlSystemEventSetFree(version, setHandle):
+    """
+    Free an NVML system event set.
+    :param version: The version of the request.
+    :param setHandle: A ctypes pointer to a c_nvmlSystemEventSet_t structure.
+    """
+    if version != nvmlSystemEventSetFreeRequest_v1:
+        raise NVMLError(NVML_ERROR_ARGUMENT_VERSION_MISMATCH)
+
+    req = c_nvmlSystemEventSetFreeRequest_v1_t()
+    req.version = version
+    req.set = setHandle
+
+    c_nvmlSystemEventSetFree(byref(req))
+
+
+def nvmlSystemRegisterEvents(version, eventTypes, setHandle):
+    """
+    Register events for an NVML system event set.
+    :param version: The version of the request.
+    :param eventTypes: A bitmask of event types to register.
+    :param setHandle: A ctypes pointer to a c_nvmlSystemEventSet_t structure.
+    """
+    if version != nvmlSystemRegisterEventRequest_v1:
+        raise NVMLError(NVML_ERROR_ARGUMENT_VERSION_MISMATCH)
+
+    req = c_nvmlSystemRegisterEventRequest_v1_t()
+    req.version = version
+    req.eventTypes = eventTypes
+    req.set = setHandle
+
+    c_nvmlSystemRegisterEvents(byref(req))
+
+
+def nvmlSystemEventSetWait(version, setHandle, timeoutMs, eventData, eventDataSize):
+    """
+    Wait for events in an NVML system event set.
+    :param version: The version of the request.
+    :param setHandle: A ctypes pointer to a c_nvmlSystemEventSet_t structure.
+    :param timeoutMs: Timeout in milliseconds.
+    :param eventData: A ctypes array of c_nvmlSystemEventData_v1_t for event data.
+    :param eventDataSize: Number of c_nvmlSystemEventData_v1_t in evenData array..
+    :return: A list of dictionaries containing event data.
+    """
+    if version != nvmlSystemEventSetWaitRequest_v1:
+        raise NVMLError(NVML_ERROR_ARGUMENT_VERSION_MISMATCH)
+
+    req = c_nvmlSystemEventSetWaitRequest_v1_t()
+    req.version = version
+    req.timeoutms = timeoutMs
+    req.set = setHandle
+    req.data = eventData
+    req.dataSize = eventDataSize
+
+    ret = c_nvmlSystemEventSetWait(byref(req))
+    if ret == NVML_ERROR_TIMEOUT:
+        return []  # Timeout, no events occurred.
+
+    # Extract event data from the buffer
+    events = []
+    for i in range(req.numEvent):
+        events.append({
+            'eventType': eventData[i].eventType,
+            'gpuId': eventData[i].gpuId
+        })
+    return events
+
+class c_nvmlAccountingStats_t(_PrintableStructure):
+    _fields_ = [
+        ('gpuUtilization', c_uint),
+        ('memoryUtilization', c_uint),
+        ('maxMemoryUsage', c_ulonglong),
+        ('time', c_ulonglong),
+        ('startTime', c_ulonglong),
+        ('isRunning', c_uint),
+        ('reserved', c_uint * 5)
+    ]
+
+class c_nvmlVgpuVersion_t(Structure):
+    _fields_ = [("minVersion", c_uint),
+                ("maxVersion", c_uint)
+               ]
+
+class c_nvmlVgpuMetadata_t(_PrintableStructure):
+    _fields_ = [("version", c_uint),
+                ("revision", c_uint),
+                ("guestInfoState", _nvmlVgpuGuestInfoState_t),
+                ("guestDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE),
+                ("hostDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE),
+                ("reserved", c_uint * 6),
+                ("vgpuVirtualizationCaps", c_uint),
+                ("guestVgpuVersion", c_uint),
+                ("opaqueDataSize", c_uint),
+                ("opaqueData", c_char * NVML_VGPU_METADATA_OPAQUE_DATA_SIZE)
+               ]
+
+class c_nvmlVgpuPgpuMetadata_t(_PrintableStructure):
+    _fields_ = [("version", c_uint),
+                ("revision", c_uint),
+                ("hostDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE),
+                ("pgpuVirtualizationCaps", c_uint),
+                ("reserved", c_uint * 5),
+                ("hostSupportedVgpuRange", c_nvmlVgpuVersion_t),
+                ("opaqueDataSize", c_uint),
+                ("opaqueData", c_char * NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE)
+               ]
+
+class c_nvmlVgpuPgpuCompatibility_t(Structure):
+    _fields_ = [("vgpuVmCompatibility", _nvmlVgpuVmCompatibility_t),
+                ("compatibilityLimitCode", _nvmlVgpuPgpuCompatibilityLimitCode_t)
+               ]
+
+## vGPU scheduler policy defines
+NVML_VGPU_SCHEDULER_POLICY_UNKNOWN      = 0
+NVML_VGPU_SCHEDULER_POLICY_BEST_EFFORT  = 1
+NVML_VGPU_SCHEDULER_POLICY_EQUAL_SHARE  = 2
+NVML_VGPU_SCHEDULER_POLICY_FIXED_SHARE  = 3
+
+## Supported vGPU scheduler policy count
+NVML_SUPPORTED_VGPU_SCHEDULER_POLICY_COUNT  = 3
+
+NVML_SCHEDULER_SW_MAX_LOG_ENTRIES           = 200
+
+NVML_VGPU_SCHEDULER_ARR_DEFAULT   = 0
+NVML_VGPU_SCHEDULER_ARR_DISABLE   = 1
+NVML_VGPU_SCHEDULER_ARR_ENABLE    = 2
+
+NVML_VGPU_SCHEDULER_ENGINE_TYPE_GRAPHICS    = 1
+
+class c_nvmlVgpuSchedDataWithARR_t(_PrintableStructure):
+    _fields_ = [
+        ('avgFactor',   c_uint),
+        ('timeslice',   c_uint),
+    ]
+
+class c_nvmlVgpuSchedData_t(_PrintableStructure):
+    _fields_ = [
+        ('timeslice',   c_uint),
+    ]
+
+class c_nvmlVgpuSchedulerParams_t(Union):
+    _fields_ = [
+        ('vgpuSchedDataWithARR', c_nvmlVgpuSchedDataWithARR_t),
+        ('vgpuSchedData',        c_nvmlVgpuSchedData_t),
+    ]
+
+class c_nvmlVgpuSchedulerLogEntry_t(_PrintableStructure):
+    _fields_ = [
+        ('timestamp',                   c_ulonglong),
+        ('timeRunTotal',                c_ulonglong),
+        ('timeRun',                     c_ulonglong),
+        ('swRunlistId',                 c_uint),
+        ('targetTimeSlice',             c_ulonglong),
+        ('cumulativePreemptionTime',    c_ulonglong),
+    ]
+
+class c_nvmlVgpuSchedulerLog_t(_PrintableStructure):
+    _fields_ = [
+        ('engineId',        c_uint),
+        ('schedulerPolicy', c_uint),
+        ('arrMode',         c_uint),
+        ('schedulerParams', c_nvmlVgpuSchedulerParams_t),
+        ('entriesCount',    c_uint),
+        ('logEntries',      c_nvmlVgpuSchedulerLogEntry_t * NVML_SCHEDULER_SW_MAX_LOG_ENTRIES),
+    ]
+
+class c_nvmlVgpuSchedulerGetState_t(_PrintableStructure):
+    _fields_ = [
+        ('schedulerPolicy', c_uint),
+        ('arrMode',         c_uint),
+        ('schedulerParams', c_nvmlVgpuSchedulerParams_t),
+    ]
+
+class c_nvmlVgpuSchedSetDataWithARR_t(_PrintableStructure):
+    _fields_ = [
+        ('avgFactor',   c_uint),
+        ('frequency',   c_uint),
+    ]
+
+class c_nvmlVgpuSchedSetData_t(_PrintableStructure):
+    _fields_ = [
+        ('timeslice',   c_uint),
+    ]
+
+class c_nvmlVgpuSchedulerSetParams_t(Union):
+    _fields_ = [
+        ('vgpuSchedDataWithARR', c_nvmlVgpuSchedSetDataWithARR_t),
+        ('vgpuSchedData',        c_nvmlVgpuSchedSetData_t),
+    ]
+
+class c_nvmlVgpuSchedulerSetState_t(_PrintableStructure):
+    _fields_ = [
+        ('schedulerPolicy', c_uint),
+        ('enableARRMode',   c_uint),
+        ('schedulerParams', c_nvmlVgpuSchedulerSetParams_t),
+    ]
+
+class c_nvmlVgpuSchedulerCapabilities_t(_PrintableStructure):
+    _fields_ = [
+        ('supportedSchedulers', c_uint * NVML_SUPPORTED_VGPU_SCHEDULER_POLICY_COUNT),
+        ('maxTimeslice',        c_uint),
+        ('minTimeslice',        c_uint),
+        ('isArrModeSupported',  c_uint),
+        ('maxFrequencyForARR',  c_uint),
+        ('minFrequencyForARR',  c_uint),
+        ('maxAvgFactorForARR',  c_uint),
+        ('minAvgFactorForARR',  c_uint),
+    ]
+
+class c_nvmlVgpuTypeIdInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('vgpuCount', c_uint),
+        ('vgpuTypeIds', POINTER(c_uint)),
+    ]
+
+nvmlVgpuTypeIdInfo_v1 = 0x1000010
+
+class c_nvmlVgpuTypeMaxInstance_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('vgpuTypeId', c_uint),
+        ('maxInstancePerGI', c_uint),
+    ]
+
+nvmlVgpuTypeMaxInstance_v1 = 0x100000C
+
+class c_nvmlActiveVgpuInstanceInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('vgpuCount', c_uint),
+        ('vgpuInstances', POINTER(c_uint)),
+    ]
+
+nvmlActiveVgpuInstanceInfo_v1 = 0x1000010
+
+class c_nvmlVgpuSchedulerState_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('engineId', c_uint),
+        ('schedulerPolicy', c_uint),
+        ('enableARRMode', c_uint),
+        ('schedulerParams', c_nvmlVgpuSchedulerSetParams_t),
+    ]
+
+nvmlVgpuSchedulerState_v1 = 0x1000018
+
+class c_nvmlVgpuSchedulerStateInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version',         c_uint),                        # input
+        ('engineId',        c_uint),                        # input. One of NVML_ENGINE_TYPE*
+        ('schedulerPolicy', c_uint),                        # output
+        ('arrMode',         c_uint),                        # output
+        ('schedulerParams', c_nvmlVgpuSchedulerParams_t),   # output
+    ]
+
+nvmlVgpuSchedulerStateInfo_v1 = 0x1000018
+
+class c_nvmlVgpuSchedulerLogInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version',         c_uint),                                                            # input
+        ('engineId',        c_uint),                                                            # input. One of NVML_ENGINE_TYPE*
+        ('schedulerPolicy', c_uint),                                                            # output
+        ('arrMode',         c_uint),                                                            # output
+        ('schedulerParams', c_nvmlVgpuSchedulerParams_t),                                       # output
+        ('entriesCount',    c_uint),                                                            # output
+        ('logEntries',      c_nvmlVgpuSchedulerLogEntry_t * NVML_SCHEDULER_SW_MAX_LOG_ENTRIES), # output
+    ]
+
+nvmlVgpuSchedulerLogInfo_v1 = 0x10025A0
+
+class c_nvmlVgpuCreatablePlacementInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('vgpuTypeId', c_uint),
+        ('count', c_uint),
+        ('placementIds', POINTER(c_uint)),
+        ('placementSize', c_uint),
+    ]
+
+nvmlVgpuCreatablePlacementInfo_v1 = 0x1000020
+
+class c_nvmlFBCStats_t(Structure):
+    _fields_ = [("sessionsCount", c_uint),
+                ("averageFPS", c_uint),
+                ("averageLatency", c_uint)
+               ]
+
+class c_nvmlFBCSession_t(_PrintableStructure):
+    _fields_ = [
+        ('sessionId', c_uint),
+        ('pid', c_uint),
+        ('vgpuInstance', _nvmlVgpuInstance_t),
+        ('displayOrdinal', c_uint),
+        ('sessionType', c_uint),
+        ('sessionFlags', c_uint),
+        ('hMaxResolution', c_uint),
+        ('vMaxResolution', c_uint),
+        ('hResolution', c_uint),
+        ('vResolution', c_uint),
+        ('averageFPS', c_uint),
+        ('averageLatency', c_uint),
+    ]
+
+NVML_DEVICE_MIG_DISABLE = 0x0
+NVML_DEVICE_MIG_ENABLE  = 0x1
+
+NVML_GPU_INSTANCE_PROFILE_1_SLICE      = 0x0
+NVML_GPU_INSTANCE_PROFILE_2_SLICE      = 0x1
+NVML_GPU_INSTANCE_PROFILE_3_SLICE      = 0x2
+NVML_GPU_INSTANCE_PROFILE_4_SLICE      = 0x3
+NVML_GPU_INSTANCE_PROFILE_7_SLICE      = 0x4
+NVML_GPU_INSTANCE_PROFILE_8_SLICE      = 0x5
+NVML_GPU_INSTANCE_PROFILE_6_SLICE      = 0x6
+NVML_GPU_INSTANCE_PROFILE_1_SLICE_REV1 = 0x7
+NVML_GPU_INSTANCE_PROFILE_2_SLICE_REV1 = 0x8
+NVML_GPU_INSTANCE_PROFILE_1_SLICE_REV2 = 0x9
+NVML_GPU_INSTANCE_PROFILE_1_SLICE_GFX    = 0xA
+NVML_GPU_INSTANCE_PROFILE_2_SLICE_GFX    = 0xB
+NVML_GPU_INSTANCE_PROFILE_4_SLICE_GFX    = 0xC
+NVML_GPU_INSTANCE_PROFILE_1_SLICE_NO_ME  = 0xD
+NVML_GPU_INSTANCE_PROFILE_2_SLICE_NO_ME  = 0xE
+NVML_GPU_INSTANCE_PROFILE_1_SLICE_ALL_ME = 0xF
+NVML_GPU_INSTANCE_PROFILE_2_SLICE_ALL_ME = 0x10
+NVML_GPU_INSTANCE_PROFILE_COUNT          = 0x11
+
+class c_nvmlGpuInstancePlacement_t(Structure):
+    _fields_ = [("start", c_uint),
+                ("size", c_uint)
+               ]
+
+class c_nvmlGpuInstanceProfileInfo_t(Structure):
+    _fields_ = [("id", c_uint),
+                ("isP2pSupported", c_uint),
+                ("sliceCount", c_uint),
+                ("instanceCount", c_uint),
+                ("multiprocessorCount", c_uint),
+                ("copyEngineCount", c_uint),
+                ("decoderCount", c_uint),
+                ("encoderCount", c_uint),
+                ("jpegCount", c_uint),
+                ("ofaCount", c_uint),
+                ("memorySizeMB", c_ulonglong),
+               ]
+
+nvmlGpuInstanceProfileInfo_v2 = 0x02000098
+
+class c_nvmlGpuInstanceProfileInfo_v2_t(_PrintableStructure):
+    _fields_ = [("version", c_uint),
+                ("id", c_uint),
+                ("isP2pSupported", c_uint),
+                ("sliceCount", c_uint),
+                ("instanceCount", c_uint),
+                ("multiprocessorCount", c_uint),
+                ("copyEngineCount", c_uint),
+                ("decoderCount", c_uint),
+                ("encoderCount", c_uint),
+                ("jpegCount", c_uint),
+                ("ofaCount", c_uint),
+                ("memorySizeMB", c_ulonglong),
+                ("name", c_char * NVML_DEVICE_NAME_V2_BUFFER_SIZE)
+               ]
+    
+    def __init__(self):
+        super(c_nvmlGpuInstanceProfileInfo_v2_t, self).__init__(version=nvmlGpuInstanceProfileInfo_v2)
+
+class c_nvmlGpuInstanceInfo_t(Structure):
+    _fields_ = [("device", c_nvmlDevice_t),
+                ("id", c_uint),
+                ("profileId", c_uint),
+                ("placement", c_nvmlGpuInstancePlacement_t)
+               ]
+
+class struct_c_nvmlGpuInstance_t(Structure):
+    pass # opaque handle
+c_nvmlGpuInstance_t = POINTER(struct_c_nvmlGpuInstance_t)
+
+NVML_COMPUTE_INSTANCE_PROFILE_1_SLICE      = 0x0
+NVML_COMPUTE_INSTANCE_PROFILE_2_SLICE      = 0x1
+NVML_COMPUTE_INSTANCE_PROFILE_3_SLICE      = 0x2
+NVML_COMPUTE_INSTANCE_PROFILE_4_SLICE      = 0x3
+NVML_COMPUTE_INSTANCE_PROFILE_7_SLICE      = 0x4
+NVML_COMPUTE_INSTANCE_PROFILE_8_SLICE      = 0x5
+NVML_COMPUTE_INSTANCE_PROFILE_6_SLICE      = 0x6
+NVML_COMPUTE_INSTANCE_PROFILE_1_SLICE_REV1 = 0x7
+NVML_COMPUTE_INSTANCE_PROFILE_COUNT        = 0x8
+
+NVML_COMPUTE_INSTANCE_ENGINE_PROFILE_SHARED = 0x0
+NVML_COMPUTE_INSTANCE_ENGINE_PROFILE_COUNT = 0x1
+
+class c_nvmlComputeInstancePlacement_t(Structure):
+    _fields_ = [("start", c_uint),
+                ("size", c_uint)
+               ]
+
+class c_nvmlComputeInstanceProfileInfo_t(Structure):
+    _fields_ = [("id", c_uint),
+                ("sliceCount", c_uint),
+                ("instanceCount", c_uint),
+                ("multiprocessorCount", c_uint),
+                ("sharedCopyEngineCount", c_uint),
+                ("sharedDecoderCount", c_uint),
+                ("sharedEncoderCount", c_uint),
+                ("sharedJpegCount", c_uint),
+                ("sharedOfaCount", c_uint)
+               ]
+
+nvmlComputeInstanceProfileInfo_v2 = 0x02000088
+
+class c_nvmlComputeInstanceProfileInfo_v2_t(_PrintableStructure):
+    _fields_ = [("version", c_uint),
+                ("id", c_uint),
+                ("sliceCount", c_uint),
+                ("instanceCount", c_uint),
+                ("multiprocessorCount", c_uint),
+                ("sharedCopyEngineCount", c_uint),
+                ("sharedDecoderCount", c_uint),
+                ("sharedEncoderCount", c_uint),
+                ("sharedJpegCount", c_uint),
+                ("sharedOfaCount", c_uint),
+                ("name", c_char * NVML_DEVICE_NAME_V2_BUFFER_SIZE)
+               ]
+
+    def __init__(self):
+        super(c_nvmlComputeInstanceProfileInfo_v2_t, self).__init__(version=nvmlComputeInstanceProfileInfo_v2)
+
+class c_nvmlComputeInstanceInfo_t(Structure):
+    _fields_ = [("device", c_nvmlDevice_t),
+                ("gpuInstance", c_nvmlGpuInstance_t),
+                ("id", c_uint),
+                ("profileId", c_uint),
+                ("placement", c_nvmlComputeInstancePlacement_t)
+               ]
+
+NVML_MAX_GPU_UTILIZATIONS = 8
+NVML_GPU_UTILIZATION_DOMAIN_GPU    = 0
+NVML_GPU_UTILIZATION_DOMAIN_FB     = 1
+NVML_GPU_UTILIZATION_DOMAIN_VID    = 2
+NVML_GPU_UTILIZATION_DOMAIN_BUS    = 3
+class c_nvmlGpuDynamicPstatesUtilization_t(Structure):
+    _fields_ = [("bIsPresent", c_uint, 1),
+                ("percentage", c_uint),
+                ("incThreshold", c_uint),
+                ("decThreshold", c_uint)]
+class c_nvmlGpuDynamicPstatesInfo_t(Structure):
+    _fields_ = [("flags", c_uint),
+                ("utilization", c_nvmlGpuDynamicPstatesUtilization_t * NVML_MAX_GPU_UTILIZATIONS)]
+
+NVML_MAX_THERMAL_SENSORS_PER_GPU = 3
+
+NVML_THERMAL_TARGET_NONE          = 0
+NVML_THERMAL_TARGET_GPU           = 1
+NVML_THERMAL_TARGET_MEMORY        = 2
+NVML_THERMAL_TARGET_POWER_SUPPLY  = 4
+NVML_THERMAL_TARGET_BOARD         = 8
+NVML_THERMAL_TARGET_VCD_BOARD     = 9
+NVML_THERMAL_TARGET_VCD_INLET     = 10
+NVML_THERMAL_TARGET_VCD_OUTLET    = 11
+NVML_THERMAL_TARGET_ALL           = 15
+NVML_THERMAL_TARGET_UNKNOWN       = -1
+
+NVML_THERMAL_CONTROLLER_NONE            = 0
+NVML_THERMAL_CONTROLLER_GPU_INTERNAL    = 1
+NVML_THERMAL_CONTROLLER_ADM1032         = 2
+NVML_THERMAL_CONTROLLER_ADT7461         = 3
+NVML_THERMAL_CONTROLLER_MAX6649         = 4
+NVML_THERMAL_CONTROLLER_MAX1617         = 5
+NVML_THERMAL_CONTROLLER_LM99            = 6
+NVML_THERMAL_CONTROLLER_LM89            = 7
+NVML_THERMAL_CONTROLLER_LM64            = 8
+NVML_THERMAL_CONTROLLER_G781            = 9
+NVML_THERMAL_CONTROLLER_ADT7473         = 10
+NVML_THERMAL_CONTROLLER_SBMAX6649       = 11
+NVML_THERMAL_CONTROLLER_VBIOSEVT        = 12
+NVML_THERMAL_CONTROLLER_OS              = 13
+NVML_THERMAL_CONTROLLER_NVSYSCON_CANOAS = 14
+NVML_THERMAL_CONTROLLER_NVSYSCON_E551   = 15
+NVML_THERMAL_CONTROLLER_MAX6649R        = 16
+NVML_THERMAL_CONTROLLER_ADT7473S        = 17
+NVML_THERMAL_CONTROLLER_UNKNOWN         = -1
+
+class c_nvmlGpuThermalSensor_t(Structure):
+    _fields_ = [("controller", c_int),
+                ("defaultMinTemp", c_int),
+                ("defaultMaxTemp", c_int),
+                ("currentTemp", c_int),
+                ("target", c_int)]
+class c_nvmlGpuThermalSettings_t(Structure):
+    _fields_ = [("count", c_uint),
+                ("sensor", c_nvmlGpuThermalSensor_t * NVML_MAX_THERMAL_SENSORS_PER_GPU)]
+
+_nvmlCoolerControl_t = c_uint
+NVML_THERMAL_COOLER_SIGNAL_NONE        = 0
+NVML_THERMAL_COOLER_SIGNAL_TOGGLE      = 1
+NVML_THERMAL_COOLER_SIGNAL_VARIABLE    = 2
+NVML_THERMAL_COOLER_SIGNAL_COUNT       = 3
+
+_nvmlCoolerTarget_t = c_uint
+NVML_THERMAL_COOLER_TARGET_NONE          = (1 << 0)
+NVML_THERMAL_COOLER_TARGET_GPU           = (1 << 1)
+NVML_THERMAL_COOLER_TARGET_MEMORY        = (1 << 2)
+NVML_THERMAL_COOLER_TARGET_POWER_SUPPLY  = (1 << 3)
+NVML_THERMAL_COOLER_TARGET_GPU_RELATED   = (NVML_THERMAL_COOLER_TARGET_GPU | NVML_THERMAL_COOLER_TARGET_MEMORY | NVML_THERMAL_COOLER_TARGET_POWER_SUPPLY)
+
+class c_nvmlCoolerInfo_t(_PrintableStructure):
+    _fields_ = [("version", c_uint),
+                ("index", c_uint),
+                ("coolerControlType", _nvmlCoolerControl_t),
+                ("coolerTarget", _nvmlCoolerTarget_t)
+               ]
+
+nvmlCoolerInfo_v1 = 0x1000010
+
+def nvmlDeviceGetCoolerInfo(handle):
+    c_coolerInfo = c_nvmlCoolerInfo_t()
+    c_coolerInfo.version = nvmlCoolerInfo_v1
+    c_coolerInfo.index = 0
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCoolerInfo")
+    ret = fn(handle, byref(c_coolerInfo))
+    _nvmlCheckReturn(ret)
+    return [c_coolerInfo.coolerControlType, c_coolerInfo.coolerTarget]
+
+class struct_c_nvmlComputeInstance_t(Structure):
+    pass # opaque handle
+c_nvmlComputeInstance_t = POINTER(struct_c_nvmlComputeInstance_t)
+
+class c_nvmlDeviceAttributes(Structure):
+    _fields_ = [("multiprocessorCount", c_uint),
+                ("sharedCopyEngineCount", c_uint),
+                ("sharedDecoderCount", c_uint),
+                ("sharedEncoderCount", c_uint),
+                ("sharedJpegCount", c_uint),
+                ("sharedOfaCount", c_uint),
+                ("gpuInstanceSliceCount", c_uint),
+                ("computeInstanceSliceCount", c_uint),
+                ("memorySizeMB", c_ulonglong),
+               ]
+
+class c_nvmlRowRemapperHistogramValues(Structure):
+    _fields_ = [("max", c_uint),
+                ("high", c_uint),
+                ("partial", c_uint),
+                ("low", c_uint),
+                ("none", c_uint)
+               ]
+
+NVML_GPU_CERT_CHAIN_SIZE                = 0x1000
+NVML_GPU_ATTESTATION_CERT_CHAIN_SIZE    = 0x1400
+NVML_CC_GPU_CEC_NONCE_SIZE              = 0x20
+NVML_CC_GPU_ATTESTATION_REPORT_SIZE     = 0x2000
+NVML_CC_GPU_CEC_ATTESTATION_REPORT_SIZE = 0x1000
+NVML_CC_CEC_ATTESTATION_REPORT_NOT_PRESENT = 0
+NVML_CC_CEC_ATTESTATION_REPORT_PRESENT     = 1
+
+class c_nvmlConfComputeSystemState_t(Structure):
+    _fields_ = [('environment', c_uint),
+                ('ccFeature', c_uint),
+                ('devToolsMode', c_uint),
+               ]
+
+nvmlSystemConfComputeSettings_v1 = 0x1000014
+
+class c_nvmlSystemConfComputeSettings_v1_t(Structure):
+    _fields_ = [('version', c_uint),
+                ('environment', c_uint),
+                ('ccFeature', c_uint),
+                ('devToolsMode', c_uint),
+                ('multiGpuMode', c_uint),
+               ]
+    def __init__(self):
+        super(c_nvmlSystemConfComputeSettings_v1_t, self).__init__(version=nvmlSystemConfComputeSettings_v1)
+
+class c_nvmlConfComputeSystemCaps_t(Structure):
+    _fields_ = [('cpuCaps', c_uint),
+                ('gpusCaps', c_uint),
+               ]
+
+class c_nvmlConfComputeMemSizeInfo_t(Structure):
+    _fields_ = [('protectedMemSizeKib', c_ulonglong),
+                ('unprotectedMemSizeKib', c_ulonglong),
+               ]
+
+class c_nvmlConfComputeGpuCertificate_t(Structure):
+    _fields_ = [('certChainSize', c_uint),
+                ('attestationCertChainSize', c_uint),
+                ('certChain', c_uint8 * NVML_GPU_CERT_CHAIN_SIZE),
+                ('attestationCertChain', c_uint8 * NVML_GPU_ATTESTATION_CERT_CHAIN_SIZE),
+               ]
+
+class c_nvmlConfComputeGpuAttestationReport_t(Structure):
+    _fields_ = [('isCecAttestationReportPresent', c_uint),                                   # output
+                ('attestationReportSize', c_uint),                                           # output
+                ('cecAttestationReportSize', c_uint),                                        # output
+                ('nonce', c_uint8 * NVML_CC_GPU_CEC_NONCE_SIZE),                             # input: spdm supports 32 bytes on nonce
+                ('attestationReport', c_uint8 * NVML_CC_GPU_ATTESTATION_REPORT_SIZE),        # output
+                ('cecAttestationReport', c_uint8 * NVML_CC_GPU_CEC_ATTESTATION_REPORT_SIZE), # output
+               ]
+
+class c_nvmlConfComputeSetKeyRotationThresholdInfo_t(Structure):
+    _fields_ = [('version', c_uint),
+                ('maxAttackerAdvantage', c_ulong),
+               ]
+ConfComputeSetKeyRotationThresholdInfo_v1 = 0x1000010
+
+class c_nvmlConfComputeGetKeyRotationThresholdInfo_t(Structure):
+    _fields_ = [('version', c_uint),
+                ('attackerAdvantage', c_ulong),
+               ]
+ConfComputeGetKeyRotationThresholdInfo_v1 = 0x1000010
+
+## string/bytes conversion for ease of use
+def convertStrBytes(func):
+    '''
+    In python 3, strings are unicode instead of bytes, and need to be converted for ctypes
+    Args from caller: (1, 'string', <__main__.c_nvmlDevice_t at 0xFFFFFFFF>)
+    Args passed to function: (1, b'string', <__main__.c_nvmlDevice_t at 0xFFFFFFFF)>
+    ----
+    Returned from function: b'returned string'
+    Returned to caller: 'returned string'
+    '''
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        # encoding a str returns bytes in python 2 and 3
+        args = [arg.encode() if isinstance(arg, str) else arg for arg in args]
+        res = func(*args, **kwargs)
+        # In python 2, str and bytes are the same
+        # In python 3, str is unicode and should be decoded.
+        # Ctypes handles most conversions, this only effects c_char and char arrays.
+        if isinstance(res, bytes):
+            if isinstance(res, str):
+                return res
+            return res.decode()
+        return res
+
+    if sys.version_info >= (3,):
+        return wrapper
+    return func
+
+def throwOnVersionMismatch(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except NVMLError_FunctionNotFound:
+            raise NVMLLibraryMismatchError("Unversioned function called and the "
+                                           "pyNVML version does not match the NVML lib version. "
+                                           "Either use matching pyNVML and NVML lib versions or "
+                                           "use a versioned function such as " + func.__name__ + "_v2")
+    return wrapper
+
+## C function wrappers ##
+def nvmlInitWithFlags(flags):
+    _LoadNvmlLibrary()
+
+    #
+    # Initialize the library
+    #
+    fn = _nvmlGetFunctionPointer("nvmlInitWithFlags")
+    ret = fn(flags)
+    _nvmlCheckReturn(ret)
+
+    # Atomically update refcount
+    global _nvmlLib_refcount
+    libLoadLock.acquire()
+    _nvmlLib_refcount += 1
+    libLoadLock.release()
+    return None
+
+def nvmlInit():
+    nvmlInitWithFlags(0)
+    return None
+
+def _LoadNvmlLibrary():
+    '''
+    Load the library if it isn't loaded already
+    '''
+    global nvmlLib
+
+    if (nvmlLib == None):
+        # lock to ensure only one caller loads the library
+        libLoadLock.acquire()
+
+        try:
+            # ensure the library still isn't loaded
+            if (nvmlLib == None):
+                try:
+                    if (sys.platform[:3] == "win"):
+                        # cdecl calling convention
+                        try:
+                            # Check for nvml.dll in System32 first for DCH drivers
+                            nvmlLib = CDLL(os.path.join(os.getenv("WINDIR", "C:/Windows"), "System32/nvml.dll"))
+                        except OSError as ose:
+                            # If nvml.dll is not found in System32, it should be in ProgramFiles
+                            # load nvml.dll from %ProgramFiles%/NVIDIA Corporation/NVSMI/nvml.dll
+                            nvmlLib = CDLL(os.path.join(os.getenv("ProgramFiles", "C:/Program Files"), "NVIDIA Corporation/NVSMI/nvml.dll"))
+                    else:
+                        # assume linux
+                        nvmlLib = CDLL("libnvidia-ml.so.1")
+                except OSError as ose:
+                    _nvmlCheckReturn(NVML_ERROR_LIBRARY_NOT_FOUND)
+                if (nvmlLib == None):
+                    _nvmlCheckReturn(NVML_ERROR_LIBRARY_NOT_FOUND)
+        finally:
+            # lock is always freed
+            libLoadLock.release()
+
+def nvmlShutdown():
+    #
+    # Leave the library loaded, but shutdown the interface
+    #
+    fn = _nvmlGetFunctionPointer("nvmlShutdown")
+    ret = fn()
+    _nvmlCheckReturn(ret)
+
+    # Atomically update refcount
+    global _nvmlLib_refcount
+    libLoadLock.acquire()
+    if (0 < _nvmlLib_refcount):
+        _nvmlLib_refcount -= 1
+    libLoadLock.release()
+    return None
+
+# Added in 2.285
+@convertStrBytes
+def nvmlErrorString(result):
+    fn = _nvmlGetFunctionPointer("nvmlErrorString")
+    fn.restype = c_char_p # otherwise return is an int
+    ret = fn(result)
+    return ret
+
+# Added in 2.285
+@convertStrBytes
+def nvmlSystemGetNVMLVersion():
+    c_version = create_string_buffer(NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE)
+    fn = _nvmlGetFunctionPointer("nvmlSystemGetNVMLVersion")
+    ret = fn(c_version, c_uint(NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE))
+    _nvmlCheckReturn(ret)
+    return c_version.value
+
+def nvmlSystemGetCudaDriverVersion():
+    c_cuda_version = c_int()
+    fn = _nvmlGetFunctionPointer("nvmlSystemGetCudaDriverVersion")
+    ret = fn(byref(c_cuda_version))
+    _nvmlCheckReturn(ret)
+    return c_cuda_version.value
+
+def nvmlSystemGetCudaDriverVersion_v2():
+    c_cuda_version = c_int()
+    fn = _nvmlGetFunctionPointer("nvmlSystemGetCudaDriverVersion_v2")
+    ret = fn(byref(c_cuda_version))
+    _nvmlCheckReturn(ret)
+    return c_cuda_version.value
+
+# Added in 2.285
+@convertStrBytes
+def nvmlSystemGetProcessName(pid):
+    c_name = create_string_buffer(1024)
+    fn = _nvmlGetFunctionPointer("nvmlSystemGetProcessName")
+    ret = fn(c_uint(pid), c_name, c_uint(1024))
+    _nvmlCheckReturn(ret)
+    return c_name.value
+
+@convertStrBytes
+def nvmlSystemGetDriverVersion():
+    c_version = create_string_buffer(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE)
+    fn = _nvmlGetFunctionPointer("nvmlSystemGetDriverVersion")
+    ret = fn(c_version, c_uint(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE))
+    _nvmlCheckReturn(ret)
+    return c_version.value
+
+# Added in 2.285
+def nvmlSystemGetHicVersion():
+    c_count = c_uint(0)
+    hics = None
+    fn = _nvmlGetFunctionPointer("nvmlSystemGetHicVersion")
+
+    # get the count
+    ret = fn(byref(c_count), None)
+
+    # this should only fail with insufficient size
+    if ((ret != NVML_SUCCESS) and
+        (ret != NVML_ERROR_INSUFFICIENT_SIZE)):
+        raise NVMLError(ret)
+
+    # If there are no hics
+    if (c_count.value == 0):
+        return []
+
+    hic_array = c_nvmlHwbcEntry_t * c_count.value
+    hics = hic_array()
+    ret = fn(byref(c_count), hics)
+    _nvmlCheckReturn(ret)
+    return hics
+
+def nvmlSystemGetDriverBranch():
+    c_branchInfo = c_nvmlSystemDriverBranchInfo_v1_t(0)
+    c_branchInfo.version = SystemDriverBranchInfo_v1
+    fn  = _nvmlGetFunctionPointer("nvmlSystemGetDriverBranch")
+    ret = fn(byref(c_branchInfo), c_uint(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE))
+    _nvmlCheckReturn(ret)
+    return c_branchInfo
+
+## Unit get functions
+def nvmlUnitGetCount():
+    c_count = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlUnitGetCount")
+    ret = fn(byref(c_count))
+    _nvmlCheckReturn(ret)
+    return c_count.value
+
+def nvmlUnitGetHandleByIndex(index):
+    c_index = c_uint(index)
+    unit = c_nvmlUnit_t()
+    fn = _nvmlGetFunctionPointer("nvmlUnitGetHandleByIndex")
+    ret = fn(c_index, byref(unit))
+    _nvmlCheckReturn(ret)
+    return unit
+
+def nvmlUnitGetUnitInfo(unit):
+    c_info = c_nvmlUnitInfo_t()
+    fn = _nvmlGetFunctionPointer("nvmlUnitGetUnitInfo")
+    ret = fn(unit, byref(c_info))
+    _nvmlCheckReturn(ret)
+    return c_info
+
+def nvmlUnitGetLedState(unit):
+    c_state =  c_nvmlLedState_t()
+    fn = _nvmlGetFunctionPointer("nvmlUnitGetLedState")
+    ret = fn(unit, byref(c_state))
+    _nvmlCheckReturn(ret)
+    return c_state
+
+def nvmlUnitGetPsuInfo(unit):
+    c_info = c_nvmlPSUInfo_t()
+    fn = _nvmlGetFunctionPointer("nvmlUnitGetPsuInfo")
+    ret = fn(unit, byref(c_info))
+    _nvmlCheckReturn(ret)
+    return c_info
+
+def nvmlUnitGetTemperature(unit, type):
+    c_temp = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlUnitGetTemperature")
+    ret = fn(unit, c_uint(type), byref(c_temp))
+    _nvmlCheckReturn(ret)
+    return c_temp.value
+
+def nvmlUnitGetFanSpeedInfo(unit):
+    c_speeds = c_nvmlUnitFanSpeeds_t()
+    fn = _nvmlGetFunctionPointer("nvmlUnitGetFanSpeedInfo")
+    ret = fn(unit, byref(c_speeds))
+    _nvmlCheckReturn(ret)
+    return c_speeds
+
+# added to API
+def nvmlUnitGetDeviceCount(unit):
+    c_count = c_uint(0)
+    # query the unit to determine device count
+    fn = _nvmlGetFunctionPointer("nvmlUnitGetDevices")
+    ret = fn(unit, byref(c_count), None)
+    if (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        ret = NVML_SUCCESS
+    _nvmlCheckReturn(ret)
+    return c_count.value
+
+def nvmlUnitGetDevices(unit):
+    c_count = c_uint(nvmlUnitGetDeviceCount(unit))
+    device_array = c_nvmlDevice_t * c_count.value
+    c_devices = device_array()
+    fn = _nvmlGetFunctionPointer("nvmlUnitGetDevices")
+    ret = fn(unit, byref(c_count), c_devices)
+    _nvmlCheckReturn(ret)
+    return c_devices
+
+## Device get functions
+def nvmlDeviceGetCount():
+    c_count = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCount_v2")
+    ret = fn(byref(c_count))
+    _nvmlCheckReturn(ret)
+    return c_count.value
+
+def nvmlDeviceGetHandleByIndex(index):
+    c_index = c_uint(index)
+    device = c_nvmlDevice_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleByIndex_v2")
+    ret = fn(c_index, byref(device))
+    _nvmlCheckReturn(ret)
+    return device
+
+@convertStrBytes
+def nvmlDeviceGetHandleBySerial(serial):
+    c_serial = c_char_p(serial)
+    device = c_nvmlDevice_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleBySerial")
+    ret = fn(c_serial, byref(device))
+    _nvmlCheckReturn(ret)
+    return device
+
+@convertStrBytes
+def nvmlDeviceGetHandleByUUID(uuid):
+    c_uuid = c_char_p(uuid)
+    device = c_nvmlDevice_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleByUUID")
+    ret = fn(c_uuid, byref(device))
+    _nvmlCheckReturn(ret)
+    return device
+
+@convertStrBytes
+def nvmlDeviceGetHandleByUUIDV(uuid, type):
+    c_uuid = c_nvmlUUID_t()
+    c_uuid.type = type
+    if type == NVML_UUID_TYPE_ASCII:
+        c_uuid.value.str = uuid
+    elif type == NVML_UUID_TYPE_BINARY:
+        memmove(c_uuid.value.bytes, uuid, NVML_DEVICE_UUID_BINARY_LEN)
+    device = c_nvmlDevice_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleByUUIDV")
+    ret = fn(byref(c_uuid), byref(device))
+    _nvmlCheckReturn(ret)
+    return device
+
+@convertStrBytes
+def nvmlDeviceGetHandleByPciBusId(pciBusId):
+    c_busId = c_char_p(pciBusId)
+    device = c_nvmlDevice_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetHandleByPciBusId_v2")
+    ret = fn(c_busId, byref(device))
+    _nvmlCheckReturn(ret)
+    return device
+
+@convertStrBytes
+def nvmlDeviceGetName(handle):
+    c_name = create_string_buffer(NVML_DEVICE_NAME_V2_BUFFER_SIZE)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetName")
+    ret = fn(handle, c_name, c_uint(NVML_DEVICE_NAME_V2_BUFFER_SIZE))
+    _nvmlCheckReturn(ret)
+    return c_name.value
+
+class c_nvmlDevicePerfModes_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('str', c_char * NVML_PERF_MODES_BUFFER_SIZE),
+    ]
+
+nvmlDevicePerfModes_v1 = 0x1000804
+
+@convertStrBytes
+def nvmlDeviceGetPerformanceModes(handle):
+    perfModes = c_nvmlDevicePerfModes_v1_t()
+    perfModes.version = nvmlDevicePerfModes_v1
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPerformanceModes")
+    ret = fn(handle, byref(perfModes))
+    _nvmlCheckReturn(ret)
+    return perfModes.str
+
+class c_nvmlDeviceCurrentClockFreqs_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('str', c_char * NVML_PERF_MODES_BUFFER_SIZE),
+    ]
+
+nvmlDeviceCurrentClockFreqs_v1 = 0x1000804
+
+@convertStrBytes
+def nvmlDeviceGetCurrentClockFreqs(handle):
+    currentClockFreqs = c_nvmlDeviceCurrentClockFreqs_v1_t()
+    currentClockFreqs.version = nvmlDeviceCurrentClockFreqs_v1
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrentClockFreqs")
+    ret = fn(handle, byref(currentClockFreqs))
+    _nvmlCheckReturn(ret)
+    return currentClockFreqs.str
+
+def nvmlDeviceGetBoardId(handle):
+    c_id = c_uint();
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetBoardId")
+    ret = fn(handle, byref(c_id))
+    _nvmlCheckReturn(ret)
+    return c_id.value
+
+def nvmlDeviceGetMultiGpuBoard(handle):
+    c_multiGpu = c_uint();
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMultiGpuBoard")
+    ret = fn(handle, byref(c_multiGpu))
+    _nvmlCheckReturn(ret)
+    return c_multiGpu.value
+
+def nvmlDeviceGetBrand(handle):
+    c_type = _nvmlBrandType_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetBrand")
+    ret = fn(handle, byref(c_type))
+    _nvmlCheckReturn(ret)
+    return c_type.value
+
+def nvmlDeviceGetC2cModeInfoV1(handle):
+    c_info = c_nvmlC2cModeInfo_v1_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetC2cModeInfoV")
+    ret = fn(handle, byref(c_info))
+    _nvmlCheckReturn(ret)
+    return c_info
+
+def nvmlDeviceGetC2cModeInfoV(handle):
+    return nvmlDeviceGetC2cModeInfoV1(handle)
+
+@convertStrBytes
+def nvmlDeviceGetBoardPartNumber(handle):
+    c_part_number = create_string_buffer(NVML_DEVICE_PART_NUMBER_BUFFER_SIZE)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetBoardPartNumber")
+    ret = fn(handle, c_part_number, c_uint(NVML_DEVICE_PART_NUMBER_BUFFER_SIZE))
+    _nvmlCheckReturn(ret)
+    return c_part_number.value
+
+@convertStrBytes
+def nvmlDeviceGetSerial(handle):
+    c_serial = create_string_buffer(NVML_DEVICE_SERIAL_BUFFER_SIZE)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSerial")
+    ret = fn(handle, c_serial, c_uint(NVML_DEVICE_SERIAL_BUFFER_SIZE))
+    _nvmlCheckReturn(ret)
+    return c_serial.value
+
+def nvmlDeviceGetModuleId(handle, moduleId=c_uint()):
+    isReference = type(moduleId) is not c_uint
+    moduleIdRef = moduleId if isReference else byref(moduleId)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetModuleId")
+    ret = fn(handle, moduleIdRef)
+    if isReference:
+        return ret
+    else:
+        _nvmlCheckReturn(ret)
+        return moduleId.value
+
+def nvmlDeviceGetMemoryAffinity(handle, nodeSetSize, scope):
+    affinity_array = c_ulonglong * nodeSetSize
+    c_affinity = affinity_array()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryAffinity")
+    ret = fn(handle, nodeSetSize, byref(c_affinity), _nvmlAffinityScope_t(scope))
+    _nvmlCheckReturn(ret)
+    return c_affinity
+
+def nvmlDeviceGetCpuAffinityWithinScope(handle, cpuSetSize, scope):
+    affinity_array = c_ulonglong * cpuSetSize
+    c_affinity = affinity_array()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCpuAffinityWithinScope")
+    ret = fn(handle, cpuSetSize, byref(c_affinity), _nvmlAffinityScope_t(scope))
+    _nvmlCheckReturn(ret)
+    return c_affinity
+
+def nvmlDeviceGetCpuAffinity(handle, cpuSetSize):
+    affinity_array = c_ulonglong * cpuSetSize
+    c_affinity = affinity_array()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCpuAffinity")
+    ret = fn(handle, cpuSetSize, byref(c_affinity))
+    _nvmlCheckReturn(ret)
+    return c_affinity
+
+def nvmlDeviceSetCpuAffinity(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetCpuAffinity")
+    ret = fn(handle)
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceClearCpuAffinity(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceClearCpuAffinity")
+    ret = fn(handle)
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceGetNumaNodeId(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNumaNodeId")
+    node = c_int()
+    ret = fn(handle, byref(node))
+    _nvmlCheckReturn(ret)
+    return node.value
+
+def nvmlDeviceGetMinorNumber(handle):
+    c_minor_number = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMinorNumber")
+    ret = fn(handle, byref(c_minor_number))
+    _nvmlCheckReturn(ret)
+    return c_minor_number.value
+
+@convertStrBytes
+def nvmlDeviceGetUUID(handle):
+    c_uuid = create_string_buffer(NVML_DEVICE_UUID_V2_BUFFER_SIZE)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetUUID")
+    ret = fn(handle, c_uuid, c_uint(NVML_DEVICE_UUID_V2_BUFFER_SIZE))
+    _nvmlCheckReturn(ret)
+    return c_uuid.value
+
+@convertStrBytes
+def nvmlDeviceGetInforomVersion(handle, infoRomObject):
+    c_version = create_string_buffer(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetInforomVersion")
+    ret = fn(handle, _nvmlInforomObject_t(infoRomObject),
+                 c_version, c_uint(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE))
+    _nvmlCheckReturn(ret)
+    return c_version.value
+
+# Added in 4.304
+@convertStrBytes
+def nvmlDeviceGetInforomImageVersion(handle):
+    c_version = create_string_buffer(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetInforomImageVersion")
+    ret = fn(handle, c_version, c_uint(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE))
+    _nvmlCheckReturn(ret)
+    return c_version.value
+
+# Added in 4.304
+def nvmlDeviceGetInforomConfigurationChecksum(handle):
+    c_checksum = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetInforomConfigurationChecksum")
+    ret = fn(handle, byref(c_checksum))
+    _nvmlCheckReturn(ret)
+    return c_checksum.value
+
+# Added in 4.304
+def nvmlDeviceValidateInforom(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceValidateInforom")
+    ret = fn(handle)
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceGetLastBBXFlushTime(handle):
+    c_timestamp = c_ulonglong()
+    c_durationUs = c_ulong()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetLastBBXFlushTime")
+    ret = fn(handle, byref(c_timestamp), byref(c_durationUs))
+    _nvmlCheckReturn(ret)
+    return [c_timestamp.value, c_durationUs.value]
+
+def nvmlDeviceGetDisplayMode(handle):
+    c_mode = _nvmlEnableState_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDisplayMode")
+    ret = fn(handle, byref(c_mode))
+    _nvmlCheckReturn(ret)
+    return c_mode.value
+
+def nvmlDeviceGetDisplayActive(handle):
+    c_mode = _nvmlEnableState_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDisplayActive")
+    ret = fn(handle, byref(c_mode))
+    _nvmlCheckReturn(ret)
+    return c_mode.value
+
+
+def nvmlDeviceGetPersistenceMode(handle):
+    c_state = _nvmlEnableState_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPersistenceMode")
+    ret = fn(handle, byref(c_state))
+    _nvmlCheckReturn(ret)
+    return c_state.value
+
+def nvmlDeviceGetPciInfoExt(handle, c_info):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPciInfoExt")
+    ret = fn(handle, c_info)
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceGetPciInfo_v3(handle):
+    c_info = nvmlPciInfo_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPciInfo_v3")
+    ret = fn(handle, byref(c_info))
+    _nvmlCheckReturn(ret)
+    return c_info
+
+def nvmlDeviceGetPciInfo(handle):
+    return nvmlDeviceGetPciInfo_v3(handle)
+
+def nvmlDeviceGetClockInfo(handle, type):
+    c_clock = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetClockInfo")
+    ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
+    _nvmlCheckReturn(ret)
+    return c_clock.value
+
+# Added in 2.285
+def nvmlDeviceGetMaxClockInfo(handle, type):
+    c_clock = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxClockInfo")
+    ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
+    _nvmlCheckReturn(ret)
+    return c_clock.value
+
+# Added in 4.304
+def nvmlDeviceGetApplicationsClock(handle, type):
+    c_clock = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetApplicationsClock")
+    ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
+    _nvmlCheckReturn(ret)
+    return c_clock.value
+
+def nvmlDeviceGetMaxCustomerBoostClock(handle, type):
+    c_clock = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxCustomerBoostClock")
+    ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
+    _nvmlCheckReturn(ret)
+    return c_clock.value
+
+def nvmlDeviceGetClock(handle, type, id):
+    c_clock = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetClock")
+    ret = fn(handle, _nvmlClockType_t(type), _nvmlClockId_t(id), byref(c_clock))
+    _nvmlCheckReturn(ret)
+    return c_clock.value
+
+# Added in 5.319
+def nvmlDeviceGetDefaultApplicationsClock(handle, type):
+    c_clock = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDefaultApplicationsClock")
+    ret = fn(handle, _nvmlClockType_t(type), byref(c_clock))
+    _nvmlCheckReturn(ret)
+    return c_clock.value
+
+# Added in 4.304
+def nvmlDeviceGetSupportedMemoryClocks(handle):
+    # first call to get the size
+    c_count = c_uint(0)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedMemoryClocks")
+    ret = fn(handle, byref(c_count), None)
+
+    if (ret == NVML_SUCCESS):
+        # special case, no clocks
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        clocks_array = c_uint * c_count.value
+        c_clocks = clocks_array()
+
+        # make the call again
+        ret = fn(handle, byref(c_count), c_clocks)
+        _nvmlCheckReturn(ret)
+
+        procs = []
+        for i in range(c_count.value):
+            procs.append(c_clocks[i])
+
+        return procs
+    else:
+        # error case
+        raise NVMLError(ret)
+
+# Added in 4.304
+def nvmlDeviceGetSupportedGraphicsClocks(handle, memoryClockMHz):
+    # first call to get the size
+    c_count = c_uint(0)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedGraphicsClocks")
+    ret = fn(handle, c_uint(memoryClockMHz), byref(c_count), None)
+
+    if (ret == NVML_SUCCESS):
+        # special case, no clocks
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        clocks_array = c_uint * c_count.value
+        c_clocks = clocks_array()
+
+        # make the call again
+        ret = fn(handle, c_uint(memoryClockMHz), byref(c_count), c_clocks)
+        _nvmlCheckReturn(ret)
+
+        procs = []
+        for i in range(c_count.value):
+            procs.append(c_clocks[i])
+
+        return procs
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlDeviceGetFanSpeed(handle):
+    c_speed = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanSpeed")
+    ret = fn(handle, byref(c_speed))
+    _nvmlCheckReturn(ret)
+    return c_speed.value
+
+def nvmlDeviceGetFanSpeed_v2(handle, fan):
+    c_speed = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanSpeed_v2")
+    ret = fn(handle, fan, byref(c_speed))
+    _nvmlCheckReturn(ret)
+    return c_speed.value
+
+class c_nvmlFanSpeedInfo_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('fan', c_uint),
+        ('speed', c_uint),
+    ]
+
+nvmlFanSpeedInfo_v1 = 0x100000C
+
+def nvmlDeviceGetFanSpeedRPM(handle):
+    c_fanSpeed = c_nvmlFanSpeedInfo_t()
+    c_fanSpeed.fan = 0
+    c_fanSpeed.version = nvmlFanSpeedInfo_v1
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanSpeedRPM")
+    ret = fn(handle, byref(c_fanSpeed))
+    _nvmlCheckReturn(ret)
+    return c_fanSpeed.speed
+
+def nvmlDeviceGetTargetFanSpeed(handle, fan):
+    c_speed = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTargetFanSpeed")
+    ret = fn(handle, fan, byref(c_speed))
+    _nvmlCheckReturn(ret)
+    return c_speed.value
+
+def nvmlDeviceGetNumFans(device):
+    c_numFans = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNumFans")
+    ret = fn(device, byref(c_numFans))
+    _nvmlCheckReturn(ret)
+    return c_numFans.value
+
+def nvmlDeviceSetDefaultFanSpeed_v2(handle, index):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetDefaultFanSpeed_v2");
+    ret = fn(handle, index)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceGetMinMaxFanSpeed(handle, minSpeed=c_uint(), maxSpeed=c_uint()):
+    isReference = (type(minSpeed) is not c_uint) or (type(maxSpeed) is not c_uint)
+    minSpeedRef = minSpeed if isReference else byref(minSpeed)
+    maxSpeedRef = maxSpeed if isReference else byref(maxSpeed)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMinMaxFanSpeed")
+    ret = fn(handle, minSpeedRef, maxSpeedRef)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS if isReference else [minSpeed.value, maxSpeed.value]
+
+def nvmlDeviceGetFanControlPolicy_v2(handle, fan, fanControlPolicy=c_uint()):
+    isReference = type(fanControlPolicy) is not c_uint
+    fanControlPolicyRef = fanControlPolicy if isReference else byref(fanControlPolicy)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanControlPolicy_v2")
+    ret = fn(handle, fan, fanControlPolicyRef)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS if isReference else fanControlPolicy.value
+
+def nvmlDeviceSetFanControlPolicy(handle, fan, fanControlPolicy):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetFanControlPolicy")
+    ret = fn(handle, fan, _nvmlFanControlPolicy_t(fanControlPolicy))
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+class c_nvmlTemperature_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('sensorType', _nvmlTemperatureSensors_t),
+        ('temperature', c_int),
+    ]
+nvmlTemperature_v1 = 0x100000C
+
+def nvmlDeviceGetTemperatureV1(handle, sensor):
+    c_temp = c_nvmlTemperature_v1_t()
+    c_temp.version = nvmlTemperature_v1
+    c_temp.sensorType = _nvmlTemperatureSensors_t(sensor) 
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTemperatureV")
+    ret = fn(handle, byref(c_temp))
+    _nvmlCheckReturn(ret)
+    return c_temp.temperature
+
+def nvmlDeviceGetTemperatureV(handle, sensor, version=nvmlTemperature_v1):
+    if version == nvmlTemperature_v1:
+        return nvmlDeviceGetTemperatureV1(handle, sensor)
+    else:
+        raise NVMLError(NVML_ERROR_ARGUMENT_VERSION_MISMATCH)
+
+# DEPRECATED use nvmlDeviceGetTemperatureV instead
+def nvmlDeviceGetTemperature(handle, sensor):
+    c_temp = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTemperature")
+    ret = fn(handle, _nvmlTemperatureSensors_t(sensor), byref(c_temp))
+    _nvmlCheckReturn(ret)
+    return c_temp.value
+
+def nvmlDeviceGetTemperatureThreshold(handle, threshold):
+    c_temp = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTemperatureThreshold")
+    ret = fn(handle, _nvmlTemperatureThresholds_t(threshold), byref(c_temp))
+    _nvmlCheckReturn(ret)
+    return c_temp.value
+
+def nvmlDeviceSetTemperatureThreshold(handle, threshold, temp):
+    c_temp = c_uint()
+    c_temp.value = temp
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetTemperatureThreshold")
+    ret = fn(handle, _nvmlTemperatureThresholds_t(threshold), byref(c_temp))
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceGetMarginTemperature(handle):
+    c_marginTempInfo = c_nvmlMarginTemperature_v1_t()
+    c_marginTempInfo.version = nvmlMarginTemperature_v1
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMarginTemperature")
+    ret = fn(handle, byref(c_marginTempInfo))
+    _nvmlCheckReturn(ret)
+    return c_marginTempInfo.marginTemperature
+
+# DEPRECATED use nvmlDeviceGetPerformanceState
+def nvmlDeviceGetPowerState(handle):
+    c_pstate = _nvmlPstates_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerState")
+    ret = fn(handle, byref(c_pstate))
+    _nvmlCheckReturn(ret)
+    return c_pstate.value
+
+def nvmlDeviceGetPerformanceState(handle):
+    c_pstate = _nvmlPstates_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPerformanceState")
+    ret = fn(handle, byref(c_pstate))
+    _nvmlCheckReturn(ret)
+    return c_pstate.value
+
+def nvmlDeviceGetPowerManagementMode(handle):
+    c_pcapMode = _nvmlEnableState_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementMode")
+    ret = fn(handle, byref(c_pcapMode))
+    _nvmlCheckReturn(ret)
+    return c_pcapMode.value
+
+def nvmlDeviceGetPowerManagementLimit(handle):
+    c_limit = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementLimit")
+    ret = fn(handle, byref(c_limit))
+    _nvmlCheckReturn(ret)
+    return c_limit.value
+
+# Added in 4.304
+def nvmlDeviceGetPowerManagementLimitConstraints(handle):
+    c_minLimit = c_uint()
+    c_maxLimit = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementLimitConstraints")
+    ret = fn(handle, byref(c_minLimit), byref(c_maxLimit))
+    _nvmlCheckReturn(ret)
+    return [c_minLimit.value, c_maxLimit.value]
+
+# Added in 4.304
+def nvmlDeviceGetPowerManagementDefaultLimit(handle):
+    c_limit = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementDefaultLimit")
+    ret = fn(handle, byref(c_limit))
+    _nvmlCheckReturn(ret)
+    return c_limit.value
+
+
+# Added in 331
+def nvmlDeviceGetEnforcedPowerLimit(handle):
+    c_limit = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetEnforcedPowerLimit")
+    ret = fn(handle, byref(c_limit))
+    _nvmlCheckReturn(ret)
+    return c_limit.value
+
+def nvmlDeviceGetPowerUsage(handle):
+    c_watts = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerUsage")
+    ret = fn(handle, byref(c_watts))
+    _nvmlCheckReturn(ret)
+    return c_watts.value
+
+def nvmlDeviceGetTotalEnergyConsumption(handle):
+    c_millijoules = c_uint64()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTotalEnergyConsumption")
+    ret = fn(handle, byref(c_millijoules))
+    _nvmlCheckReturn(ret)
+    return c_millijoules.value
+
+# Added in 4.304
+def nvmlDeviceGetGpuOperationMode(handle):
+    c_currState = _nvmlGpuOperationMode_t()
+    c_pendingState = _nvmlGpuOperationMode_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuOperationMode")
+    ret = fn(handle, byref(c_currState), byref(c_pendingState))
+    _nvmlCheckReturn(ret)
+    return [c_currState.value, c_pendingState.value]
+
+# Added in 4.304
+def nvmlDeviceGetCurrentGpuOperationMode(handle):
+    return nvmlDeviceGetGpuOperationMode(handle)[0]
+
+# Added in 4.304
+def nvmlDeviceGetPendingGpuOperationMode(handle):
+    return nvmlDeviceGetGpuOperationMode(handle)[1]
+
+def nvmlDeviceGetMemoryInfo(handle, version=None):
+    if not version:
+        c_memory = c_nvmlMemory_t()
+        fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryInfo")
+    else:
+        c_memory = c_nvmlMemory_v2_t()
+        c_memory.version = version
+        fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryInfo_v2")
+    ret = fn(handle, byref(c_memory))
+    _nvmlCheckReturn(ret)
+    return c_memory
+
+def nvmlDeviceGetBAR1MemoryInfo(handle):
+    c_bar1_memory = c_nvmlBAR1Memory_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetBAR1MemoryInfo")
+    ret = fn(handle, byref(c_bar1_memory))
+    _nvmlCheckReturn(ret)
+    return c_bar1_memory
+
+def nvmlDeviceGetComputeMode(handle):
+    c_mode = _nvmlComputeMode_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeMode")
+    ret = fn(handle, byref(c_mode))
+    _nvmlCheckReturn(ret)
+    return c_mode.value
+
+def nvmlDeviceGetCudaComputeCapability(handle):
+    c_major = c_int()
+    c_minor = c_int()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCudaComputeCapability")
+    ret = fn(handle, byref(c_major), byref(c_minor))
+    _nvmlCheckReturn(ret)
+    return (c_major.value, c_minor.value)
+
+def nvmlDeviceGetEccMode(handle):
+    c_currState = _nvmlEnableState_t()
+    c_pendingState = _nvmlEnableState_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetEccMode")
+    ret = fn(handle, byref(c_currState), byref(c_pendingState))
+    _nvmlCheckReturn(ret)
+    return [c_currState.value, c_pendingState.value]
+
+# added to API
+def nvmlDeviceGetCurrentEccMode(handle):
+    return nvmlDeviceGetEccMode(handle)[0]
+
+# added to API
+def nvmlDeviceGetPendingEccMode(handle):
+    return nvmlDeviceGetEccMode(handle)[1]
+
+def nvmlDeviceGetDefaultEccMode(handle):
+    c_defaultState = _nvmlEnableState_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDefaultEccMode")
+    ret = fn(handle, byref(c_defaultState))
+    _nvmlCheckReturn(ret)
+    return [c_defaultState.value]
+
+def nvmlDeviceGetTotalEccErrors(handle, errorType, counterType):
+    c_count = c_ulonglong()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTotalEccErrors")
+    ret = fn(handle, _nvmlMemoryErrorType_t(errorType),
+                 _nvmlEccCounterType_t(counterType), byref(c_count))
+    _nvmlCheckReturn(ret)
+    return c_count.value
+
+# This is deprecated, instead use nvmlDeviceGetMemoryErrorCounter
+def nvmlDeviceGetDetailedEccErrors(handle, errorType, counterType):
+    c_counts = c_nvmlEccErrorCounts_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDetailedEccErrors")
+    ret = fn(handle, _nvmlMemoryErrorType_t(errorType),
+                 _nvmlEccCounterType_t(counterType), byref(c_counts))
+    _nvmlCheckReturn(ret)
+    return c_counts
+
+# Added in 4.304
+def nvmlDeviceGetMemoryErrorCounter(handle, errorType, counterType, locationType):
+    c_count = c_ulonglong()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryErrorCounter")
+    ret = fn(handle,
+             _nvmlMemoryErrorType_t(errorType),
+             _nvmlEccCounterType_t(counterType),
+             _nvmlMemoryLocation_t(locationType),
+             byref(c_count))
+    _nvmlCheckReturn(ret)
+    return c_count.value
+
+def nvmlDeviceGetUtilizationRates(handle):
+    c_util = c_nvmlUtilization_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetUtilizationRates")
+    ret = fn(handle, byref(c_util))
+    _nvmlCheckReturn(ret)
+    return c_util
+
+def nvmlDeviceGetEncoderUtilization(handle):
+    c_util = c_uint()
+    c_samplingPeriod = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderUtilization")
+    ret = fn(handle, byref(c_util), byref(c_samplingPeriod))
+    _nvmlCheckReturn(ret)
+    return [c_util.value, c_samplingPeriod.value]
+
+def nvmlDeviceGetDecoderUtilization(handle):
+    c_util = c_uint()
+    c_samplingPeriod = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDecoderUtilization")
+    ret = fn(handle, byref(c_util), byref(c_samplingPeriod))
+    _nvmlCheckReturn(ret)
+    return [c_util.value, c_samplingPeriod.value]
+
+def nvmlDeviceGetJpgUtilization(handle):
+    c_util = c_uint()
+    c_samplingPeriod = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetJpgUtilization")
+    ret = fn(handle, byref(c_util), byref(c_samplingPeriod))
+    _nvmlCheckReturn(ret)
+    return [c_util.value, c_samplingPeriod.value]
+
+def nvmlDeviceGetOfaUtilization(handle):
+    c_util = c_uint()
+    c_samplingPeriod = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetOfaUtilization")
+    ret = fn(handle, byref(c_util), byref(c_samplingPeriod))
+    _nvmlCheckReturn(ret)
+    return [c_util.value, c_samplingPeriod.value]
+
+def nvmlDeviceGetPcieReplayCounter(handle):
+    c_replay = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieReplayCounter")
+    ret = fn(handle, byref(c_replay))
+    _nvmlCheckReturn(ret)
+    return c_replay.value
+
+def nvmlDeviceGetDriverModel(handle):
+    c_currModel = _nvmlDriverModel_t()
+    c_pendingModel = _nvmlDriverModel_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDriverModel")
+    ret = fn(handle, byref(c_currModel), byref(c_pendingModel))
+    _nvmlCheckReturn(ret)
+    return [c_currModel.value, c_pendingModel.value]
+
+# added to API
+def nvmlDeviceGetCurrentDriverModel(handle):
+    return nvmlDeviceGetDriverModel(handle)[0]
+
+# added to API
+def nvmlDeviceGetPendingDriverModel(handle):
+    return nvmlDeviceGetDriverModel(handle)[1]
+
+# Added in 2.285
+@convertStrBytes
+def nvmlDeviceGetVbiosVersion(handle):
+    c_version = create_string_buffer(NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetVbiosVersion")
+    ret = fn(handle, c_version, c_uint(NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE))
+    _nvmlCheckReturn(ret)
+    return c_version.value
+
+# Added in 2.285
+def nvmlDeviceGetComputeRunningProcesses_v2(handle):
+    # first call to get the size
+    c_count = c_uint(0)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeRunningProcesses_v2")
+    ret = fn(handle, byref(c_count), None)
+    if (ret == NVML_SUCCESS):
+        # special case, no running processes
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        # oversize the array incase more processes are created
+        c_count.value = c_count.value * 2 + 5
+        proc_array = c_nvmlProcessInfo_v2_t * c_count.value
+        c_procs = proc_array()
+        # make the call again
+        ret = fn(handle, byref(c_count), c_procs)
+        _nvmlCheckReturn(ret)
+        procs = []
+        for i in range(c_count.value):
+            # use an alternative struct for this object
+            obj = nvmlStructToFriendlyObject(c_procs[i])
+            if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value):
+                # special case for WDDM on Windows, see comment above
+                obj.usedGpuMemory = None
+            procs.append(obj)
+        return procs
+    else:
+        # error case
+        raise NVMLError(ret)
+
+# Added in 2.285
+def nvmlDeviceGetComputeRunningProcesses_v3(handle):
+    # first call to get the size
+    c_count = c_uint(0)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeRunningProcesses_v3")
+    ret = fn(handle, byref(c_count), None)
+
+    if (ret == NVML_SUCCESS):
+        # special case, no running processes
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        # oversize the array incase more processes are created
+        c_count.value = c_count.value * 2 + 5
+        proc_array = c_nvmlProcessInfo_v3_t * c_count.value
+        c_procs = proc_array()
+
+        # make the call again
+        ret = fn(handle, byref(c_count), c_procs)
+        _nvmlCheckReturn(ret)
+
+        procs = []
+        for i in range(c_count.value):
+            # use an alternative struct for this object
+            obj = nvmlStructToFriendlyObject(c_procs[i])
+            if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value):
+                # special case for WDDM on Windows, see comment above
+                obj.usedGpuMemory = None
+            procs.append(obj)
+
+        return procs
+    else:
+        # error case
+        raise NVMLError(ret)
+
+@throwOnVersionMismatch
+def nvmlDeviceGetComputeRunningProcesses(handle):
+    return nvmlDeviceGetComputeRunningProcesses_v3(handle)
+
+def nvmlDeviceGetGraphicsRunningProcesses_v2(handle):
+    # first call to get the size
+    c_count = c_uint(0)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGraphicsRunningProcesses_v2")
+    ret = fn(handle, byref(c_count), None)
+    if (ret == NVML_SUCCESS):
+        # special case, no running processes
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        # oversize the array incase more processes are created
+        c_count.value = c_count.value * 2 + 5
+        proc_array = c_nvmlProcessInfo_v2_t * c_count.value
+        c_procs = proc_array()
+        # make the call again
+        ret = fn(handle, byref(c_count), c_procs)
+        _nvmlCheckReturn(ret)
+        procs = []
+        for i in range(c_count.value):
+            # use an alternative struct for this object
+            obj = nvmlStructToFriendlyObject(c_procs[i])
+            if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value):
+                # special case for WDDM on Windows, see comment above
+                obj.usedGpuMemory = None
+            procs.append(obj)
+        return procs
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlDeviceGetGraphicsRunningProcesses_v3(handle):
+    # first call to get the size
+    c_count = c_uint(0)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGraphicsRunningProcesses_v3")
+    ret = fn(handle, byref(c_count), None)
+
+    if (ret == NVML_SUCCESS):
+        # special case, no running processes
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        # oversize the array incase more processes are created
+        c_count.value = c_count.value * 2 + 5
+        proc_array = c_nvmlProcessInfo_v3_t * c_count.value
+        c_procs = proc_array()
+
+        # make the call again
+        ret = fn(handle, byref(c_count), c_procs)
+        _nvmlCheckReturn(ret)
+
+        procs = []
+        for i in range(c_count.value):
+            # use an alternative struct for this object
+            obj = nvmlStructToFriendlyObject(c_procs[i])
+            if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value):
+                # special case for WDDM on Windows, see comment above
+                obj.usedGpuMemory = None
+            procs.append(obj)
+
+        return procs
+    else:
+        # error case
+        raise NVMLError(ret)
+
+@throwOnVersionMismatch
+def nvmlDeviceGetGraphicsRunningProcesses(handle):
+    return nvmlDeviceGetGraphicsRunningProcesses_v3(handle)
+
+@throwOnVersionMismatch
+def nvmlDeviceGetMPSComputeRunningProcesses(handle):
+    return nvmlDeviceGetMPSComputeRunningProcesses_v3(handle)
+
+def nvmlDeviceGetMPSComputeRunningProcesses_v2(handle):
+    # first call to get the size
+    c_count = c_uint(0)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMPSComputeRunningProcesses_v2")
+    ret = fn(handle, byref(c_count), None)
+
+    if (ret == NVML_SUCCESS):
+        # special case, no running processes
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        # oversize the array incase more processes are created
+        c_count.value = c_count.value * 2 + 5
+        proc_array = c_nvmlProcessInfo_v2_t * c_count.value
+        c_procs = proc_array()
+
+        # make the call again
+        ret = fn(handle, byref(c_count), c_procs)
+        _nvmlCheckReturn(ret)
+
+        procs = []
+        for i in range(c_count.value):
+            # use an alternative struct for this object
+            obj = nvmlStructToFriendlyObject(c_procs[i])
+            if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value):
+                # special case for WDDM on Windows, see comment above
+                obj.usedGpuMemory = None
+            procs.append(obj)
+
+        return procs
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlDeviceGetMPSComputeRunningProcesses_v3(handle):
+    # first call to get the size
+    c_count = c_uint(0)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMPSComputeRunningProcesses_v3")
+    ret = fn(handle, byref(c_count), None)
+
+    if (ret == NVML_SUCCESS):
+        # special case, no running processes
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        # oversize the array incase more processes are created
+        c_count.value = c_count.value * 2 + 5
+        proc_array = c_nvmlProcessInfo_v3_t * c_count.value
+        c_procs = proc_array()
+
+        # make the call again
+        ret = fn(handle, byref(c_count), c_procs)
+        _nvmlCheckReturn(ret)
+
+        procs = []
+        for i in range(c_count.value):
+            # use an alternative struct for this object
+            obj = nvmlStructToFriendlyObject(c_procs[i])
+            if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value):
+                # special case for WDDM on Windows, see comment above
+                obj.usedGpuMemory = None
+            procs.append(obj)
+
+        return procs
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlDeviceGetRunningProcessDetailList(handle, version, mode):
+    c_processDetailList = c_nvmlProcessDetailList_t()
+    c_processDetailList.version = version
+    c_processDetailList.mode = mode
+
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetRunningProcessDetailList")
+
+    # first call to get the size
+    ret = fn(handle, byref(c_processDetailList))
+    if (ret == NVML_SUCCESS):
+        # special case, no running processes
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        c_procs = c_nvmlProcessDetail_v1_t * c_processDetailList.numProcArrayEntries
+        c_processDetailList.procArray = cast((c_procs)(), POINTER(c_nvmlProcessDetail_v1_t))
+
+        # make the call again
+        ret = fn(handle, byref(c_processDetailList))
+        _nvmlCheckReturn(ret)
+
+        procs = []
+        for i in range(c_processDetailList.numProcArrayEntries):
+            # use an alternative struct for this object
+            obj = c_processDetailList.procArray[i]
+            if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value):
+                obj.usedGpuMemory = None
+            if (obj.usedGpuCcProtectedMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value):
+                obj.usedGpuCcProtectedMemory = None
+            procs.append(obj)
+
+        return procs
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlDeviceGetAutoBoostedClocksEnabled(handle):
+    c_isEnabled = _nvmlEnableState_t()
+    c_defaultIsEnabled = _nvmlEnableState_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAutoBoostedClocksEnabled")
+    ret = fn(handle, byref(c_isEnabled), byref(c_defaultIsEnabled))
+    _nvmlCheckReturn(ret)
+    return [c_isEnabled.value, c_defaultIsEnabled.value]
+    #Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks
+
+## Set functions
+def nvmlUnitSetLedState(unit, color):
+    fn = _nvmlGetFunctionPointer("nvmlUnitSetLedState")
+    ret = fn(unit, _nvmlLedColor_t(color))
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceSetPersistenceMode(handle, mode):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetPersistenceMode")
+    ret = fn(handle, _nvmlEnableState_t(mode))
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceSetComputeMode(handle, mode):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetComputeMode")
+    ret = fn(handle, _nvmlComputeMode_t(mode))
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceSetEccMode(handle, mode):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetEccMode")
+    ret = fn(handle, _nvmlEnableState_t(mode))
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceClearEccErrorCounts(handle, counterType):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceClearEccErrorCounts")
+    ret = fn(handle, _nvmlEccCounterType_t(counterType))
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceSetDriverModel(handle, model):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetDriverModel")
+    ret = fn(handle, _nvmlDriverModel_t(model))
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceSetAutoBoostedClocksEnabled(handle, enabled):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetAutoBoostedClocksEnabled")
+    ret = fn(handle, _nvmlEnableState_t(enabled))
+    _nvmlCheckReturn(ret)
+    return None
+    #Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks
+
+def nvmlDeviceSetDefaultAutoBoostedClocksEnabled(handle, enabled, flags):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetDefaultAutoBoostedClocksEnabled")
+    ret = fn(handle, _nvmlEnableState_t(enabled), c_uint(flags))
+    _nvmlCheckReturn(ret)
+    return None
+    #Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks
+
+def nvmlDeviceSetGpuLockedClocks(handle, minGpuClockMHz, maxGpuClockMHz):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpuLockedClocks")
+    ret = fn(handle, c_uint(minGpuClockMHz), c_uint(maxGpuClockMHz))
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceResetGpuLockedClocks(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceResetGpuLockedClocks")
+    ret = fn(handle)
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceSetMemoryLockedClocks(handle, minMemClockMHz, maxMemClockMHz):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetMemoryLockedClocks")
+    ret = fn(handle, c_uint(minMemClockMHz), c_uint(maxMemClockMHz))
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceResetMemoryLockedClocks(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceResetMemoryLockedClocks")
+    ret = fn(handle)
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceGetClkMonStatus(handle, c_clkMonInfo=nvmlClkMonStatus_t()):
+    isReference = type(c_clkMonInfo) is not nvmlClkMonStatus_t
+    c_clkMonInfoRef = c_clkMonInfo if isReference else byref(c_clkMonInfo)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetClkMonStatus")
+    ret = fn(handle, c_clkMonInfoRef)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS if isReference else c_clkMonInfo
+
+# Added in 4.304
+def nvmlDeviceSetApplicationsClocks(handle, maxMemClockMHz, maxGraphicsClockMHz):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetApplicationsClocks")
+    ret = fn(handle, c_uint(maxMemClockMHz), c_uint(maxGraphicsClockMHz))
+    _nvmlCheckReturn(ret)
+    return None
+
+# Added in 4.304
+def nvmlDeviceResetApplicationsClocks(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceResetApplicationsClocks")
+    ret = fn(handle)
+    _nvmlCheckReturn(ret)
+    return None
+
+# Added in 4.304
+def nvmlDeviceSetPowerManagementLimit(handle, limit):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetPowerManagementLimit")
+    ret = fn(handle, c_uint(limit))
+    _nvmlCheckReturn(ret)
+    return None
+
+# Added in 4.304
+def nvmlDeviceSetGpuOperationMode(handle, mode):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpuOperationMode")
+    ret = fn(handle, _nvmlGpuOperationMode_t(mode))
+    _nvmlCheckReturn(ret)
+    return None
+
+# Added in 2.285
+def nvmlEventSetCreate():
+    fn = _nvmlGetFunctionPointer("nvmlEventSetCreate")
+    eventSet = c_nvmlEventSet_t()
+    ret = fn(byref(eventSet))
+    _nvmlCheckReturn(ret)
+    return eventSet
+
+# Added in 2.285
+def nvmlDeviceRegisterEvents(handle, eventTypes, eventSet):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceRegisterEvents")
+    ret = fn(handle, c_ulonglong(eventTypes), eventSet)
+    _nvmlCheckReturn(ret)
+    return None
+
+# Added in 2.285
+def nvmlDeviceGetSupportedEventTypes(handle):
+    c_eventTypes = c_ulonglong()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedEventTypes")
+    ret = fn(handle, byref(c_eventTypes))
+    _nvmlCheckReturn(ret)
+    return c_eventTypes.value
+
+# raises NVML_ERROR_TIMEOUT exception on timeout
+def nvmlEventSetWait_v2(eventSet, timeoutms):
+    fn = _nvmlGetFunctionPointer("nvmlEventSetWait_v2")
+    data = c_nvmlEventData_t()
+    ret = fn(eventSet, byref(data), c_uint(timeoutms))
+    _nvmlCheckReturn(ret)
+    return data
+
+def nvmlEventSetWait(eventSet, timeoutms):
+    return nvmlEventSetWait_v2(eventSet, timeoutms)
+
+# Added in 2.285
+def nvmlEventSetFree(eventSet):
+    fn = _nvmlGetFunctionPointer("nvmlEventSetFree")
+    ret = fn(eventSet)
+    _nvmlCheckReturn(ret)
+    return None
+
+# Added in 3.295
+def nvmlDeviceOnSameBoard(handle1, handle2):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceOnSameBoard")
+    onSameBoard = c_int()
+    ret = fn(handle1, handle2, byref(onSameBoard))
+    _nvmlCheckReturn(ret)
+    return (onSameBoard.value != 0)
+
+# Added in 3.295
+def nvmlDeviceGetCurrPcieLinkGeneration(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrPcieLinkGeneration")
+    gen = c_uint()
+    ret = fn(handle, byref(gen))
+    _nvmlCheckReturn(ret)
+    return gen.value
+
+# Added in 3.295
+def nvmlDeviceGetMaxPcieLinkGeneration(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxPcieLinkGeneration")
+    gen = c_uint()
+    ret = fn(handle, byref(gen))
+    _nvmlCheckReturn(ret)
+    return gen.value
+
+# Added in 3.295
+def nvmlDeviceGetCurrPcieLinkWidth(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrPcieLinkWidth")
+    width = c_uint()
+    ret = fn(handle, byref(width))
+    _nvmlCheckReturn(ret)
+    return width.value
+
+# Added in 3.295
+def nvmlDeviceGetMaxPcieLinkWidth(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxPcieLinkWidth")
+    width = c_uint()
+    ret = fn(handle, byref(width))
+    _nvmlCheckReturn(ret)
+    return width.value
+
+def nvmlDeviceGetGpuMaxPcieLinkGeneration(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuMaxPcieLinkGeneration")
+    gen = c_uint()
+    ret = fn(handle, byref(gen))
+    _nvmlCheckReturn(ret)
+    return gen.value
+
+# Added in 4.304
+def nvmlDeviceGetSupportedClocksThrottleReasons(handle):
+    c_reasons= c_ulonglong()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedClocksThrottleReasons")
+    ret = fn(handle, byref(c_reasons))
+    _nvmlCheckReturn(ret)
+    return c_reasons.value
+
+def nvmlDeviceGetSupportedClocksEventReasons(handle):
+    c_reasons= c_ulonglong()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedClocksEventReasons")
+    ret = fn(handle, byref(c_reasons))
+    _nvmlCheckReturn(ret)
+    return c_reasons.value
+
+# Added in 4.304
+def nvmlDeviceGetCurrentClocksThrottleReasons(handle):
+    c_reasons= c_ulonglong()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrentClocksThrottleReasons")
+    ret = fn(handle, byref(c_reasons))
+    _nvmlCheckReturn(ret)
+    return c_reasons.value
+
+def nvmlDeviceGetCurrentClocksEventReasons(handle):
+    c_reasons= c_ulonglong()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrentClocksEventReasons")
+    ret = fn(handle, byref(c_reasons))
+    _nvmlCheckReturn(ret)
+    return c_reasons.value
+
+# Added in 5.319
+def nvmlDeviceGetIndex(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetIndex")
+    c_index = c_uint()
+    ret = fn(handle, byref(c_index))
+    _nvmlCheckReturn(ret)
+    return c_index.value
+
+# Added in 5.319
+def nvmlDeviceGetAccountingMode(handle):
+    c_mode = _nvmlEnableState_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingMode")
+    ret = fn(handle, byref(c_mode))
+    _nvmlCheckReturn(ret)
+    return c_mode.value
+
+def nvmlDeviceSetAccountingMode(handle, mode):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetAccountingMode")
+    ret = fn(handle, _nvmlEnableState_t(mode))
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceClearAccountingPids(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceClearAccountingPids")
+    ret = fn(handle)
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceGetAccountingStats(handle, pid):
+    stats = c_nvmlAccountingStats_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingStats")
+    ret = fn(handle, c_uint(pid), byref(stats))
+    _nvmlCheckReturn(ret)
+    if (stats.maxMemoryUsage == NVML_VALUE_NOT_AVAILABLE_ulonglong.value):
+        # special case for WDDM on Windows, see comment above
+        stats.maxMemoryUsage = None
+    return stats
+
+def nvmlDeviceGetAccountingPids(handle):
+    count = c_uint(nvmlDeviceGetAccountingBufferSize(handle))
+    pids = (c_uint * count.value)()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingPids")
+    ret = fn(handle, byref(count), pids)
+    _nvmlCheckReturn(ret)
+    return list(map(int, pids[0:count.value]))
+
+def nvmlDeviceGetAccountingBufferSize(handle):
+    bufferSize = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingBufferSize")
+    ret = fn(handle, byref(bufferSize))
+    _nvmlCheckReturn(ret)
+    return int(bufferSize.value)
+
+def nvmlDeviceGetRetiredPages(device, sourceFilter):
+    c_source = _nvmlPageRetirementCause_t(sourceFilter)
+    c_count = c_uint(0)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetRetiredPages")
+
+    # First call will get the size
+    ret = fn(device, c_source, byref(c_count), None)
+
+    # this should only fail with insufficient size
+    if ((ret != NVML_SUCCESS) and
+        (ret != NVML_ERROR_INSUFFICIENT_SIZE)):
+        raise NVMLError(ret)
+
+    # call again with a buffer
+    # oversize the array for the rare cases where additional pages
+    # are retired between NVML calls
+    c_count.value = c_count.value * 2 + 5
+    page_array = c_ulonglong * c_count.value
+    c_pages = page_array()
+    ret = fn(device, c_source, byref(c_count), c_pages)
+    _nvmlCheckReturn(ret)
+    return list(map(int, c_pages[0:c_count.value]))
+
+def nvmlDeviceGetRetiredPages_v2(device, sourceFilter):
+    c_source = _nvmlPageRetirementCause_t(sourceFilter)
+    c_count = c_uint(0)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetRetiredPages_v2")
+
+    # First call will get the size
+    ret = fn(device, c_source, byref(c_count), None)
+
+    # this should only fail with insufficient size
+    if ((ret != NVML_SUCCESS) and
+        (ret != NVML_ERROR_INSUFFICIENT_SIZE)):
+        raise NVMLError(ret)
+
+    # call again with a buffer
+    # oversize the array for the rare cases where additional pages
+    # are retired between NVML calls
+    c_count.value = c_count.value * 2 + 5
+    page_array = c_ulonglong * c_count.value
+    c_pages = page_array()
+    times_array = c_ulonglong * c_count.value
+    c_times = times_array()
+    ret = fn(device, c_source, byref(c_count), c_pages, c_times)
+    _nvmlCheckReturn(ret)
+    return [ { 'address': int(c_pages[i]), 'timestamp': int(c_times[i]) } for i in range(c_count.value) ];
+
+def nvmlDeviceGetRetiredPagesPendingStatus(device):
+    c_pending = _nvmlEnableState_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetRetiredPagesPendingStatus")
+    ret = fn(device, byref(c_pending))
+    _nvmlCheckReturn(ret)
+    return int(c_pending.value)
+
+def nvmlDeviceGetAPIRestriction(device, apiType):
+    c_permission = _nvmlEnableState_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAPIRestriction")
+    ret = fn(device, _nvmlRestrictedAPI_t(apiType), byref(c_permission))
+    _nvmlCheckReturn(ret)
+    return int(c_permission.value)
+
+def nvmlDeviceSetAPIRestriction(handle, apiType, isRestricted):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetAPIRestriction")
+    ret = fn(handle, _nvmlRestrictedAPI_t(apiType), _nvmlEnableState_t(isRestricted))
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceGetBridgeChipInfo(handle):
+    bridgeHierarchy = c_nvmlBridgeChipHierarchy_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetBridgeChipInfo")
+    ret = fn(handle, byref(bridgeHierarchy))
+    _nvmlCheckReturn(ret)
+    return bridgeHierarchy
+
+def nvmlDeviceGetSamples(device, sampling_type, timeStamp):
+    c_sampling_type = _nvmlSamplingType_t(sampling_type)
+    c_time_stamp = c_ulonglong(timeStamp)
+    c_sample_count = c_uint(0)
+    c_sample_value_type = _nvmlValueType_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSamples")
+
+    ## First Call gets the size
+    ret = fn(device, c_sampling_type, c_time_stamp, byref(c_sample_value_type), byref(c_sample_count), None)
+
+    # Stop if this fails
+    if (ret != NVML_SUCCESS):
+        raise NVMLError(ret)
+
+    sampleArray = c_sample_count.value * c_nvmlSample_t
+    c_samples = sampleArray()
+    ret = fn(device, c_sampling_type, c_time_stamp,  byref(c_sample_value_type), byref(c_sample_count), c_samples)
+    _nvmlCheckReturn(ret)
+    return (c_sample_value_type.value, c_samples[0:c_sample_count.value])
+
+def nvmlDeviceGetViolationStatus(device, perfPolicyType):
+    c_perfPolicy_type = _nvmlPerfPolicyType_t(perfPolicyType)
+    c_violTime = c_nvmlViolationTime_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetViolationStatus")
+
+    ## Invoke the method to get violation time
+    ret = fn(device, c_perfPolicy_type, byref(c_violTime))
+    _nvmlCheckReturn(ret)
+    return c_violTime
+
+def nvmlDeviceGetPcieThroughput(device, counter):
+    c_util = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieThroughput")
+    ret = fn(device, _nvmlPcieUtilCounter_t(counter), byref(c_util))
+    _nvmlCheckReturn(ret)
+    return c_util.value
+
+def nvmlSystemGetTopologyGpuSet(cpuNumber):
+    c_count = c_uint(0)
+    fn = _nvmlGetFunctionPointer("nvmlSystemGetTopologyGpuSet")
+
+    # First call will get the size
+    ret = fn(cpuNumber, byref(c_count), None)
+
+    if ret != NVML_SUCCESS:
+        raise NVMLError(ret)
+    # call again with a buffer
+    device_array = c_nvmlDevice_t * c_count.value
+    c_devices = device_array()
+    ret = fn(cpuNumber, byref(c_count), c_devices)
+    _nvmlCheckReturn(ret)
+    return list(c_devices[0:c_count.value])
+
+def nvmlDeviceGetTopologyNearestGpus(device, level):
+    c_count = c_uint(0)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTopologyNearestGpus")
+
+    # First call will get the size
+    ret = fn(device, level, byref(c_count), None)
+
+    if ret != NVML_SUCCESS:
+        raise NVMLError(ret)
+
+    # call again with a buffer
+    device_array = c_nvmlDevice_t * c_count.value
+    c_devices = device_array()
+    ret = fn(device, level, byref(c_count), c_devices)
+    _nvmlCheckReturn(ret)
+    return list(c_devices[0:c_count.value])
+
+def nvmlDeviceGetTopologyCommonAncestor(device1, device2):
+    c_level = _nvmlGpuTopologyLevel_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetTopologyCommonAncestor")
+    ret = fn(device1, device2, byref(c_level))
+    _nvmlCheckReturn(ret)
+    return c_level.value
+
+def nvmlDeviceGetNvLinkUtilizationCounter(device, link, counter):
+    c_rxcounter = c_ulonglong()
+    c_txcounter = c_ulonglong()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkUtilizationCounter")
+    ret = fn(device, link, counter, byref(c_rxcounter), byref(c_txcounter))
+    _nvmlCheckReturn(ret)
+    return (c_rxcounter.value, c_txcounter.value)
+
+def nvmlDeviceFreezeNvLinkUtilizationCounter(device, link, counter, freeze):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceFreezeNvLinkUtilizationCounter")
+    ret = fn(device, link, counter, freeze)
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceResetNvLinkUtilizationCounter(device, link, counter):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceResetNvLinkUtilizationCounter")
+    ret = fn(device, link, counter)
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceSetNvLinkUtilizationControl(device, link, counter, control, reset):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetNvLinkUtilizationControl")
+    ret = fn(device, link, counter, byref(control), reset)
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceGetNvLinkUtilizationControl(device, link, counter):
+    c_control = nvmlNvLinkUtilizationControl_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkUtilizationControl")
+    ret = fn(device, link, counter, byref(c_control))
+    _nvmlCheckReturn(ret)
+    return c_control
+
+def nvmlDeviceGetNvLinkCapability(device, link, capability):
+    c_capResult = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkCapability")
+    ret = fn(device, link, capability, byref(c_capResult))
+    _nvmlCheckReturn(ret)
+    return c_capResult.value
+
+def nvmlDeviceGetNvLinkErrorCounter(device, link, counter):
+    c_result = c_ulonglong()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkErrorCounter")
+    ret = fn(device, link, counter, byref(c_result))
+    _nvmlCheckReturn(ret)
+    return c_result.value
+
+def nvmlDeviceResetNvLinkErrorCounters(device, link):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceResetNvLinkErrorCounters")
+    ret = fn(device, link)
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceGetNvLinkRemotePciInfo(device, link):
+    c_pci = nvmlPciInfo_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkRemotePciInfo_v2")
+    ret = fn(device, link, byref(c_pci))
+    _nvmlCheckReturn(ret)
+    return c_pci
+
+def nvmlDeviceGetNvLinkRemoteDeviceType(handle, link):
+    c_type = _nvmlNvLinkDeviceType_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkRemoteDeviceType")
+    ret = fn(handle, link, byref(c_type))
+    _nvmlCheckReturn(ret)
+    return c_type.value
+
+def nvmlDeviceGetNvLinkState(device, link):
+    c_isActive = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkState")
+    ret = fn(device, link, byref(c_isActive))
+    _nvmlCheckReturn(ret)
+    return c_isActive.value
+
+def nvmlDeviceGetNvLinkVersion(device, link):
+    c_version = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkVersion")
+    ret = fn(device, link, byref(c_version))
+    _nvmlCheckReturn(ret)
+    return c_version.value
+
+def nvmlDeviceModifyDrainState(pciInfo, newState):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceModifyDrainState")
+    ret = fn(pointer(pciInfo), newState)
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceQueryDrainState(pciInfo):
+    c_newState = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceQueryDrainState")
+    ret = fn(pointer(pciInfo), byref(c_newState))
+    _nvmlCheckReturn(ret)
+    return c_newState.value
+
+def nvmlDeviceRemoveGpu(pciInfo):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceRemoveGpu")
+    ret = fn(pointer(pciInfo))
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceDiscoverGpus(pciInfo):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceDiscoverGpus")
+    ret = fn(pointer(pciInfo))
+    _nvmlCheckReturn(ret)
+    return None
+
+def nvmlDeviceGetFieldValues(handle, fieldIds):
+    values_arr = c_nvmlFieldValue_t * len(fieldIds)
+    values = values_arr()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetFieldValues")
+
+    for i, fieldId in enumerate(fieldIds):
+        try:
+            (values[i].fieldId, values[i].scopeId) = fieldId
+        except TypeError:
+            values[i].fieldId = fieldId
+
+    ret = fn(handle, c_int32(len(fieldIds)), byref(values))
+    _nvmlCheckReturn(ret)
+    return values
+
+def nvmlDeviceClearFieldValues(handle, fieldIds):
+    values_arr = c_nvmlFieldValue_t * len(fieldIds)
+    values = values_arr()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceClearFieldValues")
+
+    for i, fieldId in enumerate(fieldIds):
+        try:
+            (values[i].fieldId, values[i].scopeId) = fieldId
+        except TypeError:
+            values[i].fieldId = fieldId
+
+    ret = fn(handle, c_int32(len(fieldIds)), byref(values))
+    _nvmlCheckReturn(ret)
+    return values
+
+def nvmlDeviceGetVirtualizationMode(handle):
+    c_virtualization_mode = c_ulonglong()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetVirtualizationMode")
+    ret = fn(handle, byref(c_virtualization_mode))
+    _nvmlCheckReturn(ret)
+    return c_virtualization_mode.value
+
+def nvmlDeviceSetVirtualizationMode(handle, virtualization_mode):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetVirtualizationMode")
+    return fn(handle, virtualization_mode)
+
+def nvmlDeviceGetVgpuHeterogeneousMode(handle):
+    c_vgpuHeterogeneousMode = c_nvmlVgpuHeterogeneousMode_v1_t(0)
+    c_vgpuHeterogeneousMode.version = VgpuHeterogeneousMode_v1
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuHeterogeneousMode")
+    ret = fn(handle, byref(c_vgpuHeterogeneousMode))
+    _nvmlCheckReturn(ret)
+    return c_vgpuHeterogeneousMode.mode
+
+def nvmlDeviceSetVgpuHeterogeneousMode(handle, heterogeneous_mode):
+    c_vgpuHeterogeneousMode = c_nvmlVgpuHeterogeneousMode_v1_t(0)
+    c_vgpuHeterogeneousMode.version = VgpuHeterogeneousMode_v1
+    c_vgpuHeterogeneousMode.mode = heterogeneous_mode
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetVgpuHeterogeneousMode")
+    ret = fn(handle, byref(c_vgpuHeterogeneousMode))
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlVgpuInstanceGetPlacementId(vgpuInstance):
+    c_placement = c_nvmlVgpuPlacementId_v1_t(0)
+    c_placement.version = VgpuPlacementId_v1
+    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetPlacementId")
+    ret = fn(vgpuInstance, byref(c_placement))
+    _nvmlCheckReturn(ret)
+    return c_placement.placementId
+
+def nvmlDeviceGetVgpuTypeSupportedPlacements(handle, vgpuTypeId, mode=0, version=1):
+    c_max_instances = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances")
+    ret = fn(handle, vgpuTypeId, byref(c_max_instances))
+    _nvmlCheckReturn(ret)
+
+    if version == 2:
+        c_vgpu_placements = c_nvmlVgpuPlacementList_v2_t()
+        c_vgpu_placements.version = VgpuPlacementList_v2
+        c_vgpu_placements.count = c_max_instances.value
+        c_vgpu_placements.mode = mode
+    elif version == 1:
+        c_vgpu_placements = c_nvmlVgpuPlacementList_v1_t()
+        c_vgpu_placements.version = VgpuPlacementList_v1
+    else:
+        raise NVMLError(NVML_ERROR_ARGUMENT_VERSION_MISMATCH)
+
+    c_placements = c_uint * c_max_instances.value
+    c_vgpu_placements.placementIds = c_placements()
+    fn  = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuTypeSupportedPlacements")
+    ret = fn(handle, vgpuTypeId, byref(c_vgpu_placements))
+    _nvmlCheckReturn(ret)
+    return c_vgpu_placements
+
+def nvmlDeviceGetVgpuTypeCreatablePlacements(handle, vgpuTypeId, version=1):
+    c_max_instances = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances")
+    ret = fn(handle, vgpuTypeId, byref(c_max_instances))
+    _nvmlCheckReturn(ret)
+
+    if version == 2:
+        c_vgpu_placements = c_nvmlVgpuPlacementList_v2_t()
+        c_vgpu_placements.version = VgpuPlacementList_v2
+        c_vgpu_placements.count = c_max_instances.value
+    elif version == 1:
+        c_vgpu_placements = c_nvmlVgpuPlacementList_v1_t()
+        c_vgpu_placements.version = VgpuPlacementList_v1
+
+    c_placements = c_uint * c_max_instances.value
+    c_vgpu_placements.placementIds = c_placements()
+    fn  = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuTypeCreatablePlacements")
+    ret = fn(handle, vgpuTypeId, byref(c_vgpu_placements))
+    _nvmlCheckReturn(ret)
+    return c_vgpu_placements
+
+def nvmlGetVgpuDriverCapabilities(capability):
+    c_capResult = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlGetVgpuDriverCapabilities")
+    ret = fn(_nvmlVgpuDriverCapability_t(capability), byref(c_capResult))
+    _nvmlCheckReturn(ret)
+    return c_capResult.value
+
+def nvmlDeviceGetVgpuCapabilities(handle, capability):
+    c_capResult = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuCapabilities")
+    ret = fn(handle, _nvmlDeviceVgpuCapability_t(capability), byref(c_capResult))
+    _nvmlCheckReturn(ret)
+    return c_capResult.value
+
+def nvmlDeviceSetVgpuCapabilities(handle, capability, state):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetVgpuCapabilities")
+    ret = fn(handle, _nvmlDeviceVgpuCapability_t(capability), state)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceGetSupportedVgpus(handle):
+    # first call to get the size
+    c_vgpu_count = c_uint(0)
+
+    fn =  _nvmlGetFunctionPointer("nvmlDeviceGetSupportedVgpus")
+    ret = fn(handle, byref(c_vgpu_count), None)
+
+    if (ret == NVML_SUCCESS):
+        # special case, no supported vGPUs
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        vgpu_type_ids_array = _nvmlVgpuTypeId_t * c_vgpu_count.value
+        c_vgpu_type_ids = vgpu_type_ids_array()
+
+        # make the call again
+        ret = fn(handle, byref(c_vgpu_count), c_vgpu_type_ids)
+        _nvmlCheckReturn(ret)
+        vgpus = []
+        for i in range(c_vgpu_count.value):
+            vgpus.append(c_vgpu_type_ids[i])
+        return vgpus
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlDeviceGetCreatableVgpus(handle):
+    # first call to get the size
+    c_vgpu_count = c_uint(0)
+
+    fn =  _nvmlGetFunctionPointer("nvmlDeviceGetCreatableVgpus")
+    ret = fn(handle, byref(c_vgpu_count), None)
+
+    if (ret == NVML_SUCCESS):
+        # special case, no supported vGPUs
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        vgpu_type_ids_array = _nvmlVgpuTypeId_t * c_vgpu_count.value
+        c_vgpu_type_ids = vgpu_type_ids_array()
+
+        # make the call again
+        ret = fn(handle, byref(c_vgpu_count), c_vgpu_type_ids)
+        _nvmlCheckReturn(ret)
+        vgpus = []
+        for i in range(c_vgpu_count.value):
+            vgpus.append(c_vgpu_type_ids[i])
+        return vgpus
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlVgpuTypeGetGpuInstanceProfileId(vgpuTypeId):
+    c_profile_id = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetGpuInstanceProfileId")
+    ret = fn(vgpuTypeId, byref(c_profile_id))
+    _nvmlCheckReturn(ret)
+    return (c_profile_id.value)
+
+@convertStrBytes
+def nvmlVgpuTypeGetClass(vgpuTypeId):
+    c_class = create_string_buffer(NVML_DEVICE_NAME_BUFFER_SIZE)
+    c_buffer_size = c_uint(NVML_DEVICE_NAME_BUFFER_SIZE)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetClass")
+    ret = fn(vgpuTypeId, c_class, byref(c_buffer_size))
+    _nvmlCheckReturn(ret)
+    return c_class.value
+
+@convertStrBytes
+def nvmlVgpuTypeGetName(vgpuTypeId):
+    c_name = create_string_buffer(NVML_DEVICE_NAME_BUFFER_SIZE)
+    c_buffer_size = c_uint(NVML_DEVICE_NAME_BUFFER_SIZE)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetName")
+    ret = fn(vgpuTypeId, c_name, byref(c_buffer_size))
+    _nvmlCheckReturn(ret)
+    return c_name.value
+
+def nvmlVgpuTypeGetDeviceID(vgpuTypeId):
+    c_device_id    = c_ulonglong(0)
+    c_subsystem_id = c_ulonglong(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetDeviceID")
+    ret = fn(vgpuTypeId, byref(c_device_id), byref(c_subsystem_id))
+    _nvmlCheckReturn(ret)
+    return (c_device_id.value, c_subsystem_id.value)
+
+def nvmlVgpuTypeGetFramebufferSize(vgpuTypeId):
+    c_fb_size = c_ulonglong(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFramebufferSize")
+    ret = fn(vgpuTypeId, byref(c_fb_size))
+    _nvmlCheckReturn(ret)
+    return c_fb_size.value
+
+def nvmlVgpuTypeGetNumDisplayHeads(vgpuTypeId):
+    c_num_heads = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetNumDisplayHeads")
+    ret = fn(vgpuTypeId, byref(c_num_heads))
+    _nvmlCheckReturn(ret)
+    return c_num_heads.value
+
+def nvmlVgpuTypeGetResolution(vgpuTypeId):
+    c_xdim = c_uint(0)
+    c_ydim = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetResolution")
+    ret = fn(vgpuTypeId, 0, byref(c_xdim), byref(c_ydim))
+    _nvmlCheckReturn(ret)
+    return (c_xdim.value, c_ydim.value)
+
+@convertStrBytes
+def nvmlVgpuTypeGetLicense(vgpuTypeId):
+    c_license = create_string_buffer(NVML_GRID_LICENSE_BUFFER_SIZE)
+    c_buffer_size = c_uint(NVML_GRID_LICENSE_BUFFER_SIZE)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetLicense")
+    ret = fn(vgpuTypeId, c_license, c_buffer_size)
+    _nvmlCheckReturn(ret)
+    return c_license.value
+
+def nvmlVgpuTypeGetFrameRateLimit(vgpuTypeId):
+    c_frl_config = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFrameRateLimit")
+    ret = fn(vgpuTypeId, byref(c_frl_config))
+    _nvmlCheckReturn(ret)
+    return c_frl_config.value
+
+def nvmlVgpuTypeGetGspHeapSize(vgpuTypeId):
+    c_gsp_heap = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetGspHeapSize")
+    ret = fn(vgpuTypeId, byref(c_gsp_heap))
+    _nvmlCheckReturn(ret)
+    return c_gsp_heap.value
+
+def nvmlVgpuTypeGetFbReservation(vgpuTypeId):
+    c_fb_reservation = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFbReservation")
+    ret = fn(vgpuTypeId, byref(c_fb_reservation))
+    _nvmlCheckReturn(ret)
+    return c_fb_reservation.value
+
+def nvmlVgpuInstanceGetRuntimeStateSize(vgpuInstance):
+    c_runtime_state = nvmlVgpuRuntimeState_v1_t()
+    c_runtime_state.version = VgpuRuntimeState_v1
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetRuntimeStateSize")
+    ret = fn(vgpuInstance, byref(c_runtime_state))
+    _nvmlCheckReturn(ret)
+    return c_runtime_state
+
+def nvmlVgpuTypeGetMaxInstances(handle, vgpuTypeId):
+    c_max_instances = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances")
+    ret = fn(handle, vgpuTypeId, byref(c_max_instances))
+    _nvmlCheckReturn(ret)
+    return c_max_instances.value
+
+def nvmlVgpuTypeGetMaxInstancesPerVm(vgpuTypeId):
+    c_max_instances_per_vm = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstancesPerVm")
+    ret = fn(vgpuTypeId, byref(c_max_instances_per_vm))
+    _nvmlCheckReturn(ret)
+    return c_max_instances_per_vm.value
+
+def nvmlVgpuTypeGetBAR1Info(vgpuTypeId):
+    c_bar1Info = c_nvmlVgpuTypeBar1Info_v1_t(0)
+    c_bar1Info.version = VgpuTypeBar1Info_v1
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetBAR1Info")
+    ret = fn(vgpuTypeId, byref(c_bar1Info))
+    _nvmlCheckReturn(ret)
+    return c_bar1Info
+
+def nvmlDeviceGetActiveVgpus(handle):
+    # first call to get the size
+    c_vgpu_count = c_uint(0)
+
+    fn  = _nvmlGetFunctionPointer("nvmlDeviceGetActiveVgpus")
+    ret = fn(handle, byref(c_vgpu_count), None)
+
+    if (ret == NVML_SUCCESS):
+        # special case, no active vGPUs
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        vgpu_instance_array = _nvmlVgpuInstance_t * c_vgpu_count.value
+        c_vgpu_instances = vgpu_instance_array()
+
+        # make the call again
+        ret = fn(handle, byref(c_vgpu_count), c_vgpu_instances)
+        _nvmlCheckReturn(ret)
+        vgpus = []
+        for i in range(c_vgpu_count.value):
+            vgpus.append(c_vgpu_instances[i])
+        return vgpus
+    else:
+        # error case
+        raise NVMLError(ret)
+
+@convertStrBytes
+def nvmlVgpuInstanceGetVmID(vgpuInstance):
+    c_vm_id = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE)
+    c_buffer_size = c_uint(NVML_GRID_LICENSE_BUFFER_SIZE)
+    c_vm_id_type  = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetVmID")
+    ret = fn(vgpuInstance, byref(c_vm_id), c_buffer_size, byref(c_vm_id_type))
+    _nvmlCheckReturn(ret)
+    return (c_vm_id.value, c_vm_id_type.value)
+
+@convertStrBytes
+def nvmlVgpuInstanceGetUUID(vgpuInstance):
+    c_uuid = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE)
+    c_buffer_size = c_uint(NVML_DEVICE_UUID_BUFFER_SIZE)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetUUID")
+    ret = fn(vgpuInstance, byref(c_uuid), c_buffer_size)
+    _nvmlCheckReturn(ret)
+    return c_uuid.value
+
+@convertStrBytes
+def nvmlVgpuInstanceGetMdevUUID(vgpuInstance):
+    c_uuid = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE)
+    c_buffer_size = c_uint(NVML_DEVICE_UUID_BUFFER_SIZE)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetMdevUUID")
+    ret = fn(vgpuInstance, byref(c_uuid), c_buffer_size)
+    _nvmlCheckReturn(ret)
+    return c_uuid.value
+
+@convertStrBytes
+def nvmlVgpuInstanceGetVmDriverVersion(vgpuInstance):
+    c_driver_version = create_string_buffer(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE)
+    c_buffer_size = c_uint(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetVmDriverVersion")
+    ret = fn(vgpuInstance, byref(c_driver_version), c_buffer_size)
+    _nvmlCheckReturn(ret)
+    return c_driver_version.value
+
+def nvmlVgpuInstanceGetLicenseStatus(vgpuInstance):
+    c_license_status = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetLicenseStatus")
+    ret = fn(vgpuInstance, byref(c_license_status))
+    _nvmlCheckReturn(ret)
+    return c_license_status.value
+
+def nvmlVgpuInstanceGetLicenseInfo_v2(vgpuInstance):
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetLicenseInfo_v2")
+    c_license_info = c_nvmlVgpuLicenseInfo_t()
+    ret = fn(vgpuInstance, byref(c_license_info))
+    _nvmlCheckReturn(ret)
+    return c_license_info
+
+def nvmlVgpuInstanceGetLicenseInfo(vgpuInstance):
+    return nvmlVgpuInstanceGetLicenseInfo_v2(vgpuInstance)
+
+def nvmlVgpuInstanceGetFrameRateLimit(vgpuInstance):
+    c_frl = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFrameRateLimit")
+    ret = fn(vgpuInstance, byref(c_frl))
+    _nvmlCheckReturn(ret)
+    return c_frl.value
+
+def nvmlVgpuInstanceGetEccMode(vgpuInstance):
+    c_mode = _nvmlEnableState_t()
+    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEccMode")
+    ret = fn(vgpuInstance, byref(c_mode))
+    _nvmlCheckReturn(ret)
+    return c_mode.value
+
+def nvmlVgpuInstanceGetType(vgpuInstance):
+    c_vgpu_type = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetType")
+    ret = fn(vgpuInstance, byref(c_vgpu_type))
+    _nvmlCheckReturn(ret)
+    return c_vgpu_type.value
+
+def nvmlVgpuInstanceGetEncoderCapacity(vgpuInstance):
+    c_encoder_capacity = c_ulonglong(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderCapacity")
+    ret = fn(vgpuInstance, byref(c_encoder_capacity))
+    _nvmlCheckReturn(ret)
+    return c_encoder_capacity.value
+
+def nvmlVgpuInstanceSetEncoderCapacity(vgpuInstance, encoder_capacity):
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceSetEncoderCapacity")
+    return fn(vgpuInstance, encoder_capacity)
+
+def nvmlVgpuInstanceGetFbUsage(vgpuInstance):
+    c_fb_usage = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFbUsage")
+    ret = fn(vgpuInstance, byref(c_fb_usage))
+    _nvmlCheckReturn(ret)
+    return c_fb_usage.value
+
+def nvmlVgpuTypeGetCapabilities(vgpuTypeId, capability):
+    c_cap_result = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetCapabilities")
+    ret = fn(vgpuTypeId, _nvmlVgpuCapability_t(capability), byref(c_cap_result))
+    _nvmlCheckReturn(ret)
+    return (c_cap_result.value)
+
+def nvmlVgpuInstanceGetGpuInstanceId(vgpuInstance):
+    c_id = c_uint(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetGpuInstanceId")
+    ret = fn(vgpuInstance, byref(c_id))
+    _nvmlCheckReturn(ret)
+    return (c_id.value)
+
+@convertStrBytes
+def nvmlVgpuInstanceGetGpuPciId(vgpuInstance):
+    c_vgpuPciId = create_string_buffer(NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE)
+    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetGpuPciId")
+    ret = fn(vgpuInstance, c_vgpuPciId, byref(c_uint(NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE)))
+    _nvmlCheckReturn(ret)
+    return c_vgpuPciId.value
+
+def nvmlDeviceGetVgpuUtilization(handle, timeStamp):
+    # first call to get the size
+    c_vgpu_count = c_uint(0)
+    c_time_stamp = c_ulonglong(timeStamp)
+    c_sample_value_type = _nvmlValueType_t()
+
+    fn  = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuUtilization")
+    ret = fn(handle, c_time_stamp, byref(c_sample_value_type), byref(c_vgpu_count), None)
+
+    if (ret == NVML_SUCCESS):
+        # special case, no active vGPUs
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        sampleArray = c_vgpu_count.value * c_nvmlVgpuInstanceUtilizationSample_t
+        c_samples = sampleArray()
+
+        # make the call again
+        ret = fn(handle, c_time_stamp, byref(c_sample_value_type), byref(c_vgpu_count), c_samples)
+        _nvmlCheckReturn(ret)
+
+        return c_samples[0:c_vgpu_count.value]
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlDeviceGetVgpuInstancesUtilizationInfo(handle, timeStamp):
+    # first call to get the size
+    c_time_stamp = c_ulonglong(timeStamp)
+    c_vgpuUtilInfo = c_nvmlVgpuInstancesUtilizationInfo_v1_t(0)
+    c_vgpuUtilInfo.version = VgpuInstancesUtilizationInfo_v1
+    c_vgpuUtilInfo.sampleValType = _nvmlValueType_t()
+    c_vgpuUtilInfo.vgpuInstanceCount = c_uint(0)
+    c_vgpuUtilInfo.lastSeenTimeStamp = c_time_stamp
+
+    fn  = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuInstancesUtilizationInfo")
+    ret = fn(handle, byref(c_vgpuUtilInfo))
+
+    if (ret == NVML_SUCCESS):
+        # special case, no active vGPUs
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        sampleArray = c_vgpuUtilInfo.vgpuInstanceCount * c_nvmlVgpuInstanceUtilizationInfo_v1_t
+        c_samples = sampleArray()
+        c_vgpuUtilInfo.vgpuUtilArray = c_samples
+
+        # make the call again
+        ret = fn(handle, byref(c_vgpuUtilInfo))
+        _nvmlCheckReturn(ret)
+
+        return c_samples[0:c_vgpuUtilInfo.vgpuInstanceCount]
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlDeviceGetP2PStatus(device1, device2, p2pIndex):
+    c_p2pstatus = _nvmlGpuP2PStatus_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetP2PStatus")
+    ret = fn(device1, device2,p2pIndex, byref(c_p2pstatus))
+    _nvmlCheckReturn(ret)
+    return c_p2pstatus.value
+
+def nvmlDeviceGetGridLicensableFeatures_v4(handle):
+    c_get_grid_licensable_features = c_nvmlGridLicensableFeatures_v4_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGridLicensableFeatures_v4")
+    ret = fn(handle, byref(c_get_grid_licensable_features))
+    _nvmlCheckReturn(ret)
+
+    return (c_get_grid_licensable_features)
+
+def nvmlDeviceGetGridLicensableFeatures(handle):
+    return nvmlDeviceGetGridLicensableFeatures_v4(handle)
+
+def nvmlDeviceGetGspFirmwareVersion(handle, version=None):
+    isUserDefined = version is not None
+    if not isUserDefined:
+        version = (c_char * NVML_GSP_FIRMWARE_VERSION_BUF_SIZE)()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGspFirmwareVersion")
+    ret = fn(handle, version)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS if isUserDefined else version.value
+
+def nvmlDeviceGetGspFirmwareMode(handle, isEnabled=c_uint(), defaultMode=c_uint()):
+    isReference = type(isEnabled) is not c_uint
+    isEnabledRef = isEnabled if isReference else byref(isEnabled)
+    defaultModeRef = defaultMode if isReference else byref(defaultMode)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGspFirmwareMode")
+    ret = fn(handle, isEnabledRef, defaultModeRef)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS if isReference else [isEnabled.value, defaultMode.value]
+
+def nvmlDeviceGetEncoderCapacity(handle, encoderQueryType):
+    c_encoder_capacity = c_ulonglong(0)
+    c_encoderQuery_type = _nvmlEncoderQueryType_t(encoderQueryType)
+
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderCapacity")
+    ret = fn(handle, c_encoderQuery_type, byref(c_encoder_capacity))
+    _nvmlCheckReturn(ret)
+    return c_encoder_capacity.value
+
+def nvmlDeviceGetVgpuProcessUtilization(handle, timeStamp):
+    # first call to get the size
+    c_vgpu_count = c_uint(0)
+    c_time_stamp = c_ulonglong(timeStamp)
+
+    fn  = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuProcessUtilization")
+    ret = fn(handle, c_time_stamp, byref(c_vgpu_count), None)
+
+    if (ret == NVML_SUCCESS):
+        # special case, no active vGPUs
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        sampleArray = c_vgpu_count.value * c_nvmlVgpuProcessUtilizationSample_t
+        c_samples = sampleArray()
+
+        # make the call again
+        ret = fn(handle, c_time_stamp, byref(c_vgpu_count), c_samples)
+        _nvmlCheckReturn(ret)
+
+        return c_samples[0:c_vgpu_count.value]
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlDeviceGetVgpuProcessesUtilizationInfo(handle, timeStamp):
+    # first call to get the size
+    c_time_stamp = c_ulonglong(timeStamp)
+    c_vgpuProcUtilInfo = c_nvmlVgpuProcessesUtilizationInfo_v1_t(0)
+    c_vgpuProcUtilInfo.version = VgpuProcessesUtilizationInfo_v1
+    c_vgpuProcUtilInfo.vgpuProcessCount = c_uint(0)
+    c_vgpuProcUtilInfo.lastSeenTimeStamp = c_time_stamp
+
+    fn  = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuProcessesUtilizationInfo")
+    ret = fn(handle, byref(c_vgpuProcUtilInfo))
+
+    if (ret == NVML_SUCCESS):
+        # special case, no active vGPUs
+        return []
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        sampleArray = c_vgpuProcUtilInfo.vgpuProcessCount * c_nvmlVgpuProcessUtilizationInfo_v1_t
+        c_samples = sampleArray()
+        c_vgpuProcUtilInfo.vgpuProcUtilArray = c_samples
+
+        # make the call again
+        ret = fn(handle, byref(c_vgpuProcUtilInfo))
+        _nvmlCheckReturn(ret)
+
+        return c_samples[0:c_vgpuProcUtilInfo.vgpuProcessCount]
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlDeviceGetEncoderStats(handle):
+    c_encoderCount = c_ulonglong(0)
+    c_encodeFps = c_ulonglong(0)
+    c_encoderLatency = c_ulonglong(0)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderStats")
+    ret = fn(handle, byref(c_encoderCount), byref(c_encodeFps), byref(c_encoderLatency))
+    _nvmlCheckReturn(ret)
+    return (c_encoderCount.value, c_encodeFps.value, c_encoderLatency.value)
+
+def nvmlDeviceGetEncoderSessions(handle):
+    # first call to get the size
+    c_session_count = c_uint(0)
+
+    fn  = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderSessions")
+    ret = fn(handle, byref(c_session_count), None)
+
+    if (ret == NVML_SUCCESS):
+        if (c_session_count.value != 0):
+            # typical case
+            session_array = c_nvmlEncoderSession_t * c_session_count.value
+            c_sessions = session_array()
+
+            # make the call again
+            ret = fn(handle, byref(c_session_count), c_sessions)
+            _nvmlCheckReturn(ret)
+            sessions = []
+            for i in range(c_session_count.value):
+                sessions.append(c_sessions[i])
+            return sessions
+        else:
+            return []  # no active sessions
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlDeviceGetFBCStats(handle):
+    c_fbcStats = c_nvmlFBCStats_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetFBCStats")
+    ret = fn(handle, byref(c_fbcStats))
+    _nvmlCheckReturn(ret)
+    return c_fbcStats
+
+def nvmlDeviceGetFBCSessions(handle):
+    # first call to get the size
+    c_session_count = c_uint(0)
+
+    fn  = _nvmlGetFunctionPointer("nvmlDeviceGetFBCSessions")
+    ret = fn(handle, byref(c_session_count), None)
+
+    if (ret == NVML_SUCCESS):
+        if (c_session_count.value != 0):
+            # typical case
+            session_array = c_nvmlFBCSession_t * c_session_count.value
+            c_sessions = session_array()
+
+            # make the call again
+            ret = fn(handle, byref(c_session_count), c_sessions)
+            _nvmlCheckReturn(ret)
+            sessions = []
+            for i in range(c_session_count.value):
+                sessions.append(c_sessions[i])
+            return sessions
+        else:
+            return []  # no active sessions
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlVgpuInstanceGetEncoderStats(vgpuInstance):
+    c_encoderCount    = c_ulonglong(0)
+    c_encodeFps       = c_ulonglong(0)
+    c_encoderLatency  = c_ulonglong(0)
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderStats")
+    ret = fn(vgpuInstance, byref(c_encoderCount), byref(c_encodeFps), byref(c_encoderLatency))
+    _nvmlCheckReturn(ret)
+    return (c_encoderCount.value, c_encodeFps.value, c_encoderLatency.value)
+
+def nvmlVgpuInstanceGetEncoderSessions(vgpuInstance):
+    # first call to get the size
+    c_session_count = c_uint(0)
+
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderSessions")
+    ret = fn(vgpuInstance, byref(c_session_count), None)
+
+    if (ret == NVML_SUCCESS):
+        if (c_session_count.value != 0):
+            # typical case
+            session_array = c_nvmlEncoderSession_t * c_session_count.value
+            c_sessions = session_array()
+
+            # make the call again
+            ret = fn(vgpuInstance, byref(c_session_count), c_sessions)
+            _nvmlCheckReturn(ret)
+            sessions = []
+            for i in range(c_session_count.value):
+                sessions.append(c_sessions[i])
+            return sessions
+        else:
+            return []  # no active sessions
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlVgpuInstanceGetFBCStats(vgpuInstance):
+    c_fbcStats = c_nvmlFBCStats_t()
+    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFBCStats")
+    ret = fn(vgpuInstance, byref(c_fbcStats))
+    _nvmlCheckReturn(ret)
+    return c_fbcStats
+
+def nvmlVgpuInstanceGetFBCSessions(vgpuInstance):
+    # first call to get the size
+    c_session_count = c_uint(0)
+
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFBCSessions")
+    ret = fn(vgpuInstance, byref(c_session_count), None)
+
+    if (ret == NVML_SUCCESS):
+        if (c_session_count.value != 0):
+            # typical case
+            session_array = c_nvmlFBCSession_t * c_session_count.value
+            c_sessions = session_array()
+
+            # make the call again
+            ret = fn(vgpuInstance, byref(c_session_count), c_sessions)
+            _nvmlCheckReturn(ret)
+            sessions = []
+            for i in range(c_session_count.value):
+                sessions.append(c_sessions[i])
+            return sessions
+        else:
+            return []  # no active sessions
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlDeviceGetProcessUtilization(handle, timeStamp):
+    # first call to get the size
+    c_count = c_uint(0)
+    c_time_stamp = c_ulonglong(timeStamp)
+
+    fn  = _nvmlGetFunctionPointer("nvmlDeviceGetProcessUtilization")
+    ret = fn(handle, None, byref(c_count), c_time_stamp)
+
+    if (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        sampleArray = c_count.value * c_nvmlProcessUtilizationSample_t
+        c_samples = sampleArray()
+
+        # make the call again
+        ret = fn(handle, c_samples, byref(c_count), c_time_stamp)
+        _nvmlCheckReturn(ret)
+
+        return c_samples[0:c_count.value]
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlDeviceGetProcessesUtilizationInfo(handle, timeStamp):
+    # first call to get the size
+    c_time_stamp = c_ulonglong(timeStamp)
+    c_processesUtilInfo = c_nvmlProcessesUtilizationInfo_v1_t(0)
+    c_processesUtilInfo.version = ProcessesUtilizationInfo_v1
+    c_processesUtilInfo.processSamplesCount = c_uint(0)
+    c_processesUtilInfo.lastSeenTimeStamp = c_time_stamp
+
+    fn  = _nvmlGetFunctionPointer("nvmlDeviceGetProcessesUtilizationInfo")
+    ret = fn(handle, byref(c_processesUtilInfo))
+
+    if (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        sampleArray = c_processesUtilInfo.processSamplesCount * c_nvmlProcessUtilizationInfo_v1_t
+        c_samples = sampleArray()
+        c_processesUtilInfo.procUtilArray = c_samples
+
+        # make the call again
+        ret = fn(handle, byref(c_processesUtilInfo))
+        _nvmlCheckReturn(ret)
+
+        return c_samples[0:c_processesUtilInfo.processSamplesCount]
+    else:
+        # error case
+        raise NVMLError(ret)
+
+def nvmlVgpuInstanceGetMetadata(vgpuInstance):
+    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetMetadata")
+    c_vgpuMetadata = c_nvmlVgpuMetadata_t()
+    c_bufferSize = c_uint(0)
+    # Make the first NVML API call to get the c_bufferSize value.
+    # We have already allocated required buffer above.
+    ret = fn(vgpuInstance, byref(c_vgpuMetadata), byref(c_bufferSize))
+    if (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        ret = fn(vgpuInstance, byref(c_vgpuMetadata), byref(c_bufferSize))
+        _nvmlCheckReturn(ret)
+    else:
+        raise NVMLError(ret)
+    return c_vgpuMetadata
+
+def nvmlDeviceGetVgpuMetadata(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuMetadata")
+    c_vgpuPgpuMetadata = c_nvmlVgpuPgpuMetadata_t()
+    c_bufferSize = c_uint(0)
+    # Make the first NVML API call to get the c_bufferSize value.
+    # We have already allocated required buffer above.
+    ret = fn(handle, byref(c_vgpuPgpuMetadata), byref(c_bufferSize))
+    if (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        ret = fn(handle, byref(c_vgpuPgpuMetadata), byref(c_bufferSize))
+        _nvmlCheckReturn(ret)
+    else:
+        raise NVMLError(ret)
+    return c_vgpuPgpuMetadata
+
+def nvmlGetVgpuCompatibility(vgpuMetadata, pgpuMetadata):
+    fn = _nvmlGetFunctionPointer("nvmlGetVgpuCompatibility")
+    c_vgpuPgpuCompatibility = c_nvmlVgpuPgpuCompatibility_t()
+    ret = fn(byref(vgpuMetadata), byref(pgpuMetadata), byref(c_vgpuPgpuCompatibility))
+    _nvmlCheckReturn(ret)
+    return c_vgpuPgpuCompatibility
+
+@convertStrBytes
+def nvmlDeviceGetPgpuMetadataString(handle):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPgpuMetadataString")
+    c_pgpuMetadata = create_string_buffer(NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE)
+    c_bufferSize = c_uint(0)
+    # Make the first NVML API call to get the c_bufferSize value.
+    # We have already allocated required buffer above.
+    ret = fn(handle, byref(c_pgpuMetadata), byref(c_bufferSize))
+    if (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        ret = fn(handle, byref(c_pgpuMetadata), byref(c_bufferSize))
+        _nvmlCheckReturn(ret)
+    else:
+        raise NVMLError(ret)
+    return (c_pgpuMetadata.value, c_bufferSize.value)
+
+def nvmlDeviceGetVgpuSchedulerLog(handle):
+    c_vgpu_sched_log = c_nvmlVgpuSchedulerLog_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuSchedulerLog")
+    ret = fn(handle, byref(c_vgpu_sched_log))
+    _nvmlCheckReturn(ret)
+    return c_vgpu_sched_log
+
+def nvmlDeviceGetVgpuSchedulerState(handle):
+    c_vgpu_sched_state = c_nvmlVgpuSchedulerGetState_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuSchedulerState")
+    ret = fn(handle, byref(c_vgpu_sched_state))
+    _nvmlCheckReturn(ret)
+    return c_vgpu_sched_state
+
+def nvmlDeviceGetVgpuSchedulerCapabilities(handle):
+    c_vgpu_sched_caps = c_nvmlVgpuSchedulerCapabilities_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuSchedulerCapabilities")
+    ret = fn(handle, byref(c_vgpu_sched_caps))
+    _nvmlCheckReturn(ret)
+    return c_vgpu_sched_caps
+
+def nvmlDeviceSetVgpuSchedulerState(handle, sched_state):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetVgpuSchedulerState")
+    ret = fn(handle, byref(sched_state))
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlSetVgpuVersion(vgpuVersion):
+    fn = _nvmlGetFunctionPointer("nvmlSetVgpuVersion")
+    ret = fn(byref(vgpuVersion))
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlGetVgpuVersion(supported=None, current=None):
+    isUserDefined = (supported is not None) or (current is not None)
+    if not isUserDefined:
+        supported = c_nvmlVgpuVersion_t()
+        current = c_nvmlVgpuVersion_t()
+    fn = _nvmlGetFunctionPointer("nvmlGetVgpuVersion")
+    ret = fn(byref(supported), byref(current))
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS if isUserDefined else [(supported.minVersion,
+                                                supported.maxVersion),
+                                               (current.minVersion,
+                                                current.maxVersion)]
+
+def nvmlVgpuInstanceGetAccountingMode(vgpuInstance):
+    c_mode = _nvmlEnableState_t()
+    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingMode")
+    ret = fn(vgpuInstance, byref(c_mode))
+    _nvmlCheckReturn(ret)
+    return c_mode.value
+
+def nvmlVgpuInstanceGetAccountingPids(vgpuInstance):
+    c_pidCount = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingPids")
+    ret = fn(vgpuInstance, byref(c_pidCount), None)
+    if (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        sampleArray = c_pidCount.value * c_uint
+        c_pidArray = sampleArray()
+        ret = fn(vgpuInstance, byref(c_pidCount), byref(c_pidArray))
+        _nvmlCheckReturn(ret)
+    else:
+        raise NVMLError(ret)
+    return (c_pidCount, c_pidArray)
+
+def nvmlVgpuInstanceGetAccountingStats(vgpuInstance, pid):
+    c_accountingStats = c_nvmlAccountingStats_t()
+    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingStats")
+    ret = fn(vgpuInstance, pid, byref(c_accountingStats))
+    _nvmlCheckReturn(ret)
+    return c_accountingStats
+
+def nvmlVgpuInstanceClearAccountingPids(vgpuInstance):
+    fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceClearAccountingPids")
+    ret = fn(vgpuInstance)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlGpuInstanceGetCreatableVgpus(gpuInstance, c_vgpus):
+    c_vgpus.vgpuCount = 0;
+    fn =  _nvmlGetFunctionPointer("nvmlGpuInstanceGetCreatableVgpus")
+    ret = fn(gpuInstance, byref(c_vgpus))
+
+    if (ret == NVML_SUCCESS):
+        # special case, no supported vGPUs
+        return c_vgpus
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        vgpu_type_ids_array = _nvmlVgpuTypeId_t * c_vgpus.vgpuCount
+        c_vgpus.vgpuTypeIds = vgpu_type_ids_array()
+        ret = fn(gpuInstance, byref(c_vgpus))
+        _nvmlCheckReturn(ret)
+
+        return c_vgpus
+    else:
+        raise NVMLError(ret)
+
+def nvmlVgpuTypeGetMaxInstancesPerGpuInstance(vgpuTypeId):
+    c_max_instances = c_nvmlVgpuTypeMaxInstance_v1_t()
+    c_max_instances.version = nvmlVgpuTypeMaxInstance_v1
+    c_max_instances.vgpuTypeId = vgpuTypeId
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstancesPerGpuInstance")
+    ret = fn(byref(c_max_instances))
+    _nvmlCheckReturn(ret)
+    return c_max_instances.maxInstancePerGI
+
+def nvmlGpuInstanceGetActiveVgpus(gpuInstance, c_vgpu_instance_info):
+    c_vgpu_instance_info.vgpuCount = c_uint(0)
+
+    fn  = _nvmlGetFunctionPointer("nvmlGpuInstanceGetActiveVgpus")
+    ret = fn(gpuInstance, byref(c_vgpu_instance_info))
+
+    if (ret == NVML_SUCCESS):
+        # special case, no active vGPUs
+        return c_vgpu_instance_info
+    elif (ret == NVML_ERROR_INSUFFICIENT_SIZE):
+        # typical case
+        vgpu_instance_array = _nvmlVgpuInstance_t * c_vgpu_instance_info.vgpuCount
+        c_vgpu_instance_info.vgpuInstances = vgpu_instance_array()
+
+        # make the call again
+        ret = fn(gpuInstance, byref(c_vgpu_instance_info))
+        _nvmlCheckReturn(ret)
+        return c_vgpu_instance_info
+    else:
+        raise NVMLError(ret)
+
+def nvmlGpuInstanceSetVgpuSchedulerState(gpuInstance, sched_state):
+    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceSetVgpuSchedulerState")
+    ret = fn(gpuInstance, byref(sched_state))
+    _nvmlCheckReturn(ret)
+    return ret
+
+def nvmlGpuInstanceGetVgpuSchedulerState(gpuInstance, c_vgpu_sched_state_info):
+    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetVgpuSchedulerState")
+    ret = fn(gpuInstance, byref(c_vgpu_sched_state_info))
+    _nvmlCheckReturn(ret)
+    return c_vgpu_sched_state_info
+
+def nvmlGpuInstanceGetVgpuSchedulerLog(gpuInstance, c_vgpu_sched_log_info):
+    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetVgpuSchedulerLog")
+    ret = fn(gpuInstance, byref(c_vgpu_sched_log_info))
+    _nvmlCheckReturn(ret)
+    return c_vgpu_sched_log_info
+
+def nvmlGpuInstanceGetVgpuTypeCreatablePlacements(gpuInstance, c_vgpu_placements):
+    c_max_instances = c_nvmlVgpuTypeMaxInstance_v1_t()
+    c_max_instances.version = nvmlVgpuTypeMaxInstance_v1
+    c_max_instances.vgpuTypeId = c_vgpu_placements.vgpuTypeId
+    fn  = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstancesPerGpuInstance")
+    ret = fn(byref(c_max_instances))
+    _nvmlCheckReturn(ret)
+
+    c_vgpu_placements.count = c_max_instances.maxInstancePerGI
+
+    c_placements = c_uint * c_max_instances.maxInstancePerGI
+    c_vgpu_placements.placementIds = c_placements()
+    fn  = _nvmlGetFunctionPointer("nvmlGpuInstanceGetVgpuTypeCreatablePlacements")
+    ret = fn(gpuInstance, byref(c_vgpu_placements))
+    _nvmlCheckReturn(ret)
+    return c_vgpu_placements
+
+def nvmlGpuInstanceGetVgpuHeterogeneousMode(gpuInstance):
+    c_vgpuHeterogeneousMode = c_nvmlVgpuHeterogeneousMode_v1_t(0)
+    c_vgpuHeterogeneousMode.version = VgpuHeterogeneousMode_v1
+    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetVgpuHeterogeneousMode")
+    ret = fn(gpuInstance, byref(c_vgpuHeterogeneousMode))
+    _nvmlCheckReturn(ret)
+    return c_vgpuHeterogeneousMode.mode
+
+def nvmlGpuInstanceSetVgpuHeterogeneousMode(gpuInstance, heterogeneous_mode):
+    c_vgpuHeterogeneousMode = c_nvmlVgpuHeterogeneousMode_v1_t(0)
+    c_vgpuHeterogeneousMode.version = VgpuHeterogeneousMode_v1
+    c_vgpuHeterogeneousMode.mode = heterogeneous_mode
+    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceSetVgpuHeterogeneousMode")
+    ret = fn(gpuInstance, byref(c_vgpuHeterogeneousMode))
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlGetExcludedDeviceCount():
+    c_count = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlGetExcludedDeviceCount")
+    ret = fn(byref(c_count))
+    _nvmlCheckReturn(ret)
+    return c_count.value
+
+def nvmlGetExcludedDeviceInfoByIndex(index):
+    c_index = c_uint(index)
+    info = c_nvmlExcludedDeviceInfo_t()
+    fn = _nvmlGetFunctionPointer("nvmlGetExcludedDeviceInfoByIndex")
+    ret = fn(c_index, byref(info))
+    _nvmlCheckReturn(ret)
+    return info
+
+def nvmlDeviceGetHostVgpuMode(handle):
+    c_host_vgpu_mode = _nvmlHostVgpuMode_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetHostVgpuMode")
+    ret = fn(handle, byref(c_host_vgpu_mode))
+    _nvmlCheckReturn(ret)
+    return c_host_vgpu_mode.value
+
+def nvmlDeviceSetMigMode(device, mode):
+    c_activationStatus = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetMigMode")
+    ret = fn(device, mode, byref(c_activationStatus))
+    _nvmlCheckReturn(ret)
+    return c_activationStatus.value
+
+def nvmlDeviceGetMigMode(device):
+    c_currentMode = c_uint()
+    c_pendingMode = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMigMode")
+    ret = fn(device, byref(c_currentMode), byref(c_pendingMode))
+    _nvmlCheckReturn(ret)
+    return [c_currentMode.value, c_pendingMode.value]
+
+def nvmlDeviceGetGpuInstanceProfileInfo(device, profile, version=2):
+    if version == 2:
+        c_info = c_nvmlGpuInstanceProfileInfo_v2_t()
+        fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceProfileInfoV")
+    elif version == 1:
+        c_info = c_nvmlGpuInstanceProfileInfo_t()
+        fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceProfileInfo")
+    else:
+        raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND)
+    ret = fn(device, profile, byref(c_info))
+    _nvmlCheckReturn(ret)
+    return c_info
+
+# Define function alias for the API exposed by NVML
+nvmlDeviceGetGpuInstanceProfileInfoV = nvmlDeviceGetGpuInstanceProfileInfo
+
+def nvmlDeviceGetGpuInstanceRemainingCapacity(device, profileId):
+    c_count = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceRemainingCapacity")
+    ret = fn(device, profileId, byref(c_count))
+    _nvmlCheckReturn(ret)
+    return c_count.value
+
+def nvmlDeviceGetGpuInstancePossiblePlacements(device, profileId, placementsRef, countRef):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstancePossiblePlacements_v2")
+    ret = fn(device, profileId, placementsRef, countRef)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceCreateGpuInstance(device, profileId):
+    c_instance = c_nvmlGpuInstance_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceCreateGpuInstance")
+    ret = fn(device, profileId, byref(c_instance))
+    _nvmlCheckReturn(ret)
+    return c_instance
+
+def nvmlDeviceCreateGpuInstanceWithPlacement(device, profileId, placement):
+    c_instance = c_nvmlGpuInstance_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceCreateGpuInstanceWithPlacement")
+    ret = fn(device, profileId, placement, byref(c_instance))
+    _nvmlCheckReturn(ret)
+    return c_instance
+
+def nvmlGpuInstanceDestroy(gpuInstance):
+    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceDestroy")
+    ret = fn(gpuInstance)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceGetGpuInstances(device, profileId, gpuInstancesRef, countRef):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstances")
+    ret = fn(device, profileId, gpuInstancesRef, countRef)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceGetGpuInstanceById(device, gpuInstanceId):
+    c_instance = c_nvmlGpuInstance_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceById")
+    ret = fn(device, gpuInstanceId, byref(c_instance))
+    _nvmlCheckReturn(ret)
+    return c_instance
+
+def nvmlGpuInstanceGetInfo(gpuInstance):
+    c_info = c_nvmlGpuInstanceInfo_t()
+    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetInfo")
+    ret = fn(gpuInstance, byref(c_info))
+    _nvmlCheckReturn(ret)
+    return c_info
+
+def nvmlGpuInstanceGetComputeInstanceProfileInfo(device, profile, engProfile, version=2):
+    if version == 2:
+        c_info = c_nvmlComputeInstanceProfileInfo_v2_t()
+        fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceProfileInfoV")
+    elif version == 1:
+        c_info = c_nvmlComputeInstanceProfileInfo_t()
+        fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceProfileInfo")
+    else:
+        raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND) 
+    ret = fn(device, profile, engProfile, byref(c_info))
+    _nvmlCheckReturn(ret)
+    return c_info
+
+# Define function alias for the API exposed by NVML
+nvmlGpuInstanceGetComputeInstanceProfileInfoV = nvmlGpuInstanceGetComputeInstanceProfileInfo
+
+def nvmlGpuInstanceGetComputeInstanceRemainingCapacity(gpuInstance, profileId):
+    c_count = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceRemainingCapacity")
+    ret = fn(gpuInstance, profileId, byref(c_count))
+    _nvmlCheckReturn(ret)
+    return c_count.value
+
+def nvmlGpuInstanceGetComputeInstancePossiblePlacements(gpuInstance, profileId, placementsRef, countRef):
+    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstancePossiblePlacements")
+    ret = fn(gpuInstance, profileId, placementsRef, countRef)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlGpuInstanceCreateComputeInstance(gpuInstance, profileId):
+    c_instance = c_nvmlComputeInstance_t()
+    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceCreateComputeInstance")
+    ret = fn(gpuInstance, profileId, byref(c_instance))
+    _nvmlCheckReturn(ret)
+    return c_instance
+
+def nvmlGpuInstanceCreateComputeInstanceWithPlacement(gpuInstance, profileId, placement):
+    c_instance = c_nvmlComputeInstance_t()
+    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceCreateComputeInstanceWithPlacement")
+    ret = fn(gpuInstance, profileId, placement, byref(c_instance))
+    _nvmlCheckReturn(ret)
+    return c_instance
+
+def nvmlComputeInstanceDestroy(computeInstance):
+    fn = _nvmlGetFunctionPointer("nvmlComputeInstanceDestroy")
+    ret = fn(computeInstance)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlGpuInstanceGetComputeInstances(gpuInstance, profileId, computeInstancesRef, countRef):
+    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstances")
+    ret = fn(gpuInstance, profileId, computeInstancesRef, countRef)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlGpuInstanceGetComputeInstanceById(gpuInstance, computeInstanceId):
+    c_instance = c_nvmlComputeInstance_t()
+    fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceById")
+    ret = fn(gpuInstance, computeInstanceId, byref(c_instance))
+    _nvmlCheckReturn(ret)
+    return c_instance
+
+def nvmlComputeInstanceGetInfo_v2(computeInstance):
+    c_info = c_nvmlComputeInstanceInfo_t()
+    fn = _nvmlGetFunctionPointer("nvmlComputeInstanceGetInfo_v2")
+    ret = fn(computeInstance, byref(c_info))
+    _nvmlCheckReturn(ret)
+    return c_info
+
+def nvmlComputeInstanceGetInfo(computeInstance):
+    return nvmlComputeInstanceGetInfo_v2(computeInstance)
+
+def nvmlDeviceIsMigDeviceHandle(device):
+    c_isMigDevice = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceIsMigDeviceHandle")
+    ret = fn(device, byref(c_isMigDevice))
+    _nvmlCheckReturn(ret)
+    return c_isMigDevice
+
+def nvmlDeviceGetGpuInstanceId(device):
+    c_gpuInstanceId = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceId")
+    ret = fn(device, byref(c_gpuInstanceId))
+    _nvmlCheckReturn(ret)
+    return c_gpuInstanceId.value
+
+def nvmlDeviceGetComputeInstanceId(device):
+    c_computeInstanceId = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeInstanceId")
+    ret = fn(device, byref(c_computeInstanceId))
+    _nvmlCheckReturn(ret)
+    return c_computeInstanceId.value
+
+def nvmlDeviceGetMaxMigDeviceCount(device):
+    c_count = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxMigDeviceCount")
+    ret = fn(device, byref(c_count))
+    _nvmlCheckReturn(ret)
+    return c_count.value
+
+def nvmlDeviceGetMigDeviceHandleByIndex(device, index):
+    c_index = c_uint(index)
+    migDevice = c_nvmlDevice_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMigDeviceHandleByIndex")
+    ret = fn(device, c_index, byref(migDevice))
+    _nvmlCheckReturn(ret)
+    return migDevice
+
+def nvmlDeviceGetDeviceHandleFromMigDeviceHandle(migDevice):
+    device = c_nvmlDevice_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDeviceHandleFromMigDeviceHandle")
+    ret = fn(migDevice, byref(device))
+    _nvmlCheckReturn(ret)
+    return device
+
+def nvmlDeviceGetAttributes_v2(device):
+    c_attrs = c_nvmlDeviceAttributes()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAttributes_v2")
+    ret = fn(device, byref(c_attrs))
+    _nvmlCheckReturn(ret)
+    return c_attrs
+
+def nvmlDeviceGetAttributes(device):
+    return nvmlDeviceGetAttributes_v2(device)
+
+def nvmlDeviceGetRemappedRows(device):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetRemappedRows")
+    c_corr = c_uint()
+    c_unc = c_uint()
+    c_bpending = c_uint()
+    c_bfailure = c_uint()
+    ret = fn(device, byref(c_corr), byref(c_unc), byref(c_bpending), byref(c_bfailure))
+    _nvmlCheckReturn(ret)
+    return (c_corr.value, c_unc.value, c_bpending.value, c_bfailure.value)
+
+def nvmlDeviceGetRowRemapperHistogram(device):
+    c_vals = c_nvmlRowRemapperHistogramValues()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetRowRemapperHistogram")
+    ret = fn(device, byref(c_vals))
+    _nvmlCheckReturn(ret)
+    return c_vals
+
+def nvmlDeviceGetArchitecture(device):
+    arch = _nvmlDeviceArchitecture_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetArchitecture")
+    ret = fn(device, byref(arch))
+    _nvmlCheckReturn(ret)
+    return arch.value
+
+def nvmlDeviceGetBusType(device):
+    c_busType = _nvmlBusType_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetBusType")
+    ret = fn(device, byref(c_busType))
+    _nvmlCheckReturn(ret)
+    return c_busType.value
+
+def nvmlDeviceGetIrqNum(device):
+    c_irqNum = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetIrqNum")
+    ret = fn(device, byref(c_irqNum))
+    _nvmlCheckReturn(ret)
+    return c_irqNum.value
+
+def nvmlDeviceGetNumGpuCores(device):
+    c_numCores = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNumGpuCores")
+    ret = fn(device, byref(c_numCores))
+    _nvmlCheckReturn(ret)
+    return c_numCores.value
+
+def nvmlDeviceGetPowerSource(device):
+    c_powerSource = _nvmlPowerSource_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerSource")
+    ret = fn(device, byref(c_powerSource))
+    _nvmlCheckReturn(ret)
+    return c_powerSource.value
+
+def nvmlDeviceGetMemoryBusWidth(device):
+    c_memBusWidth = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryBusWidth")
+    ret = fn(device, byref(c_memBusWidth))
+    _nvmlCheckReturn(ret)
+    return c_memBusWidth.value
+
+def nvmlDeviceGetPcieLinkMaxSpeed(device):
+    c_speed = _nvmlPcieLinkMaxSpeed_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieLinkMaxSpeed")
+    ret = fn(device, byref(c_speed))
+    _nvmlCheckReturn(ret)
+    return c_speed.value
+
+def nvmlDeviceGetAdaptiveClockInfoStatus(device):
+    c_adaptiveClockInfoStatus = _nvmlAdaptiveClockInfoStatus_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetAdaptiveClockInfoStatus")
+    ret = fn(device, byref(c_adaptiveClockInfoStatus))
+    _nvmlCheckReturn(ret)
+    return c_adaptiveClockInfoStatus.value
+
+def nvmlDeviceGetPcieSpeed(device):
+    c_speed = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieSpeed")
+    ret = fn(device, byref(c_speed))
+    _nvmlCheckReturn(ret)
+    return c_speed.value
+
+def nvmlDeviceGetDynamicPstatesInfo(device, c_dynamicpstatesinfo=c_nvmlGpuDynamicPstatesInfo_t()):
+    isReference = type(c_dynamicpstatesinfo) is not c_nvmlGpuDynamicPstatesInfo_t
+    dynamicpstatesinfoRef = c_dynamicpstatesinfo if isReference else byref(c_dynamicpstatesinfo)
+
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDynamicPstatesInfo");
+    ret = fn(device, dynamicpstatesinfoRef)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS if isReference else c_dynamicpstatesinfo
+
+def nvmlDeviceSetFanSpeed_v2(handle, index, speed):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetFanSpeed_v2");
+    ret = fn(handle, index, speed)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceGetThermalSettings(device, sensorindex, c_thermalsettings=c_nvmlGpuThermalSettings_t()):
+    isReference = type(c_thermalsettings) is not c_nvmlGpuThermalSettings_t
+    thermalsettingsRef = c_thermalsettings if isReference else byref(c_thermalsettings)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetThermalSettings");
+    ret = fn(device, sensorindex, thermalsettingsRef)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS if isReference else c_thermalsettings.sensor[:]
+
+def nvmlDeviceGetMinMaxClockOfPState(device, clockType, pstate, minClockMHz=c_uint(), maxClockMHz=c_uint()):
+    isReference = (type(minClockMHz) is not c_uint) or (type(maxClockMHz) is not c_uint)
+    minClockMHzRef = minClockMHz if isReference else byref(minClockMHz)
+    maxClockMHzRef = maxClockMHz if isReference else byref(maxClockMHz)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMinMaxClockOfPState");
+    ret = fn(device, _nvmlClockType_t(clockType), _nvmlClockType_t(pstate), minClockMHzRef, maxClockMHzRef)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS if isReference else (minClockMHz.value, maxClockMHz.value)
+
+class c_nvmlClockOffset_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('type', _nvmlClockType_t),
+        ('pstate', _nvmlPstates_t),
+        ('clockOffsetMHz', c_int),
+        ('minClockOffsetMHz', c_int),
+        ('maxClockOffsetMHz', c_int),
+    ]
+
+nvmlClockOffset_v1 = 0x1000018
+
+def nvmlDeviceGetClockOffsets(device, info):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetClockOffsets");
+    ret = fn(device, info)
+    return NVML_SUCCESS
+
+def nvmlDeviceSetClockOffsets(device, info):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetClockOffsets");
+    ret = fn(device, info)
+    return NVML_SUCCESS
+
+def nvmlDeviceGetSupportedPerformanceStates(device):
+    pstates = []
+    c_count = c_uint(NVML_MAX_GPU_PERF_PSTATES)
+    c_size = sizeof(c_uint)*c_count.value
+
+    # NOTE: use 'c_uint' to represent the size of the nvmlPstate_t enumeration.
+    pstates_array = _nvmlPstates_t * c_count.value
+    c_pstates = pstates_array()
+
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedPerformanceStates")
+    ret = fn(device, c_pstates, c_size)
+    _nvmlCheckReturn(ret)
+
+    for value in c_pstates:
+        if value != NVML_PSTATE_UNKNOWN:
+            pstates.append(value)
+
+    return pstates
+
+def nvmlDeviceGetGpcClkVfOffset(device):
+    offset = c_int32()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpcClkVfOffset")
+    ret = fn(device, byref(offset))
+    _nvmlCheckReturn(ret)
+    return offset.value
+
+def nvmlDeviceSetGpcClkVfOffset(device, offset):
+    c_offset = c_int32(offset)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpcClkVfOffset")
+    ret = fn(device, c_offset)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceGetGpcClkMinMaxVfOffset(device, minOffset=c_int(), maxOffset=c_int()):
+    isReference = (type(minOffset) is not c_int) or (type(maxOffset) is not c_int)
+    minOffsetRef = minOffset if isReference else byref(minOffset)
+    maxOffsetRef = maxOffset if isReference else byref(maxOffset)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpcClkMinMaxVfOffset")
+    ret = fn(device, minOffsetRef, maxOffsetRef)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS if isReference else (minOffset.value, maxOffset.value)
+
+def nvmlDeviceGetMemClkVfOffset(device):
+    offset = c_int32()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemClkVfOffset")
+    ret = fn(device, byref(offset))
+    _nvmlCheckReturn(ret)
+    return offset.value
+
+def nvmlDeviceSetMemClkVfOffset(device, offset):
+    c_offset = c_int32(offset)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetMemClkVfOffset")
+    ret = fn(device, c_offset)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceGetMemClkMinMaxVfOffset(device, minOffset=c_int(), maxOffset=c_int()):
+    isReference = (type(minOffset) is not c_int) or (type(maxOffset) is not c_int)
+    minOffsetRef = minOffset if isReference else byref(minOffset)
+    maxOffsetRef = maxOffset if isReference else byref(maxOffset)
+
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemClkMinMaxVfOffset")
+    ret = fn(device, minOffsetRef, maxOffsetRef)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS if isReference else (minOffset.value, maxOffset.value)
+
+def nvmlSystemSetConfComputeGpusReadyState(state):
+    c_state = c_uint(state)
+    fn = _nvmlGetFunctionPointer("nvmlSystemSetConfComputeGpusReadyState")
+    ret = fn(c_state)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlSystemGetConfComputeGpusReadyState():
+    c_state = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeGpusReadyState")
+    ret = fn(byref(c_state))
+    _nvmlCheckReturn(ret)
+    return c_state.value
+
+def nvmlSystemGetConfComputeCapabilities():
+    c_ccSysCaps = c_nvmlConfComputeSystemCaps_t()
+    fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeCapabilities")
+    ret = fn(byref(c_ccSysCaps))
+    _nvmlCheckReturn(ret)
+    return c_ccSysCaps
+
+def nvmlSystemGetConfComputeState():
+    c_state = c_nvmlConfComputeSystemState_t()
+    fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeState")
+    ret = fn(byref(c_state))
+    _nvmlCheckReturn(ret)
+    return c_state
+
+def nvmlSystemGetConfComputeSettings(settings):
+    fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeSettings")
+    return fn(settings)
+
+def nvmlDeviceSetConfComputeUnprotectedMemSize(device, c_ccMemSize):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetConfComputeUnprotectedMemSize")
+    ret = fn(device, c_ccMemSize)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceGetConfComputeMemSizeInfo(device):
+    c_ccMemSize = c_nvmlConfComputeMemSizeInfo_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetConfComputeMemSizeInfo")
+    ret = fn(device, byref(c_ccMemSize))
+    _nvmlCheckReturn(ret)
+    return c_ccMemSize
+
+def nvmlDeviceGetConfComputeProtectedMemoryUsage(device):
+    c_memory = c_nvmlMemory_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetConfComputeProtectedMemoryUsage")
+    ret = fn(device, byref(c_memory))
+    _nvmlCheckReturn(ret)
+    return c_memory
+
+def nvmlDeviceGetConfComputeGpuCertificate(device):
+    c_cert = c_nvmlConfComputeGpuCertificate_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetConfComputeGpuCertificate")
+    ret = fn(device, byref(c_cert))
+    _nvmlCheckReturn(ret)
+    return c_cert
+
+def nvmlDeviceGetConfComputeGpuAttestationReport(device, c_nonce):
+    c_attestReport = c_nvmlConfComputeGpuAttestationReport_t()
+    c_nonce_arr = (c_uint8 * len(c_nonce))(*(c_nonce))
+    setattr(c_attestReport, 'nonce', c_nonce_arr)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetConfComputeGpuAttestationReport")
+    ret = fn(device, byref(c_attestReport))
+    _nvmlCheckReturn(ret)
+    return c_attestReport
+
+def nvmlSystemSetConfComputeKeyRotationThresholdInfo(max_atk_adv):
+    c_keyRotationThrInfo = c_nvmlConfComputeSetKeyRotationThresholdInfo_t(0)
+    c_keyRotationThrInfo.version = ConfComputeSetKeyRotationThresholdInfo_v1
+    c_keyRotationThrInfo.maxAttackerAdvantage = max_atk_adv
+    fn = _nvmlGetFunctionPointer("nvmlSystemSetConfComputeKeyRotationThresholdInfo")
+    ret = fn(byref(c_keyRotationThrInfo))
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlSystemGetConfComputeKeyRotationThresholdInfo():
+    c_keyRotationThrInfo = c_nvmlConfComputeGetKeyRotationThresholdInfo_t(0)
+    c_keyRotationThrInfo.version = ConfComputeGetKeyRotationThresholdInfo_v1
+    fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeKeyRotationThresholdInfo")
+    ret = fn(byref(c_keyRotationThrInfo))
+    _nvmlCheckReturn(ret)
+    return c_keyRotationThrInfo
+
+## GPM ##
+#########
+
+## Enums/defines
+
+#### GPM Metric Identifiers
+NVML_GPM_METRIC_GRAPHICS_UTIL               = 1 # Percentage of time any compute/graphics app was active on the GPU. 0.0 - 100.0
+NVML_GPM_METRIC_SM_UTIL                     = 2 # Percentage of SMs that were busy. 0.0 - 100.0
+NVML_GPM_METRIC_SM_OCCUPANCY                = 3 # Percentage of warps that were active vs theoretical maximum. 0.0 - 100.0
+NVML_GPM_METRIC_INTEGER_UTIL                = 4 # Percentage of time the GPU's SMs were doing integer operations. 0.0 - 100.0
+NVML_GPM_METRIC_ANY_TENSOR_UTIL             = 5 # Percentage of time the GPU's SMs were doing ANY tensor operations. 0.0 - 100.0
+NVML_GPM_METRIC_DFMA_TENSOR_UTIL            = 6 # Percentage of time the GPU's SMs were doing DFMA tensor operations. 0.0 - 100.0
+NVML_GPM_METRIC_HMMA_TENSOR_UTIL            = 7 # Percentage of time the GPU's SMs were doing HMMA tensor operations. 0.0 - 100.0
+NVML_GPM_METRIC_IMMA_TENSOR_UTIL            = 9 # Percentage of time the GPU's SMs were doing IMMA tensor operations. 0.0 - 100.0
+NVML_GPM_METRIC_DRAM_BW_UTIL                = 10 # Percentage of DRAM bw used vs theoretical maximum. 0.0 - 100.0
+NVML_GPM_METRIC_FP64_UTIL                   = 11 # Percentage of time the GPU's SMs were doing non-tensor FP64 math. 0.0 - 100.0
+NVML_GPM_METRIC_FP32_UTIL                   = 12 # Percentage of time the GPU's SMs were doing non-tensor FP32 math. 0.0 - 100.0
+NVML_GPM_METRIC_FP16_UTIL                   = 13 # Percentage of time the GPU's SMs were doing non-tensor FP16 math. 0.0 - 100.0
+NVML_GPM_METRIC_PCIE_TX_PER_SEC             = 20 # PCIe traffic from this GPU in MiB/sec
+NVML_GPM_METRIC_PCIE_RX_PER_SEC             = 21 # PCIe traffic to this GPU in MiB/sec
+NVML_GPM_METRIC_NVDEC_0_UTIL                = 30 # Percent utilization of NVDEC 0. 0.0 - 100.0
+NVML_GPM_METRIC_NVDEC_1_UTIL                = 31 # Percent utilization of NVDEC 1. 0.0 - 100.0
+NVML_GPM_METRIC_NVDEC_2_UTIL                = 32 # Percent utilization of NVDEC 2. 0.0 - 100.0
+NVML_GPM_METRIC_NVDEC_3_UTIL                = 33 # Percent utilization of NVDEC 3. 0.0 - 100.0
+NVML_GPM_METRIC_NVDEC_4_UTIL                = 34 # Percent utilization of NVDEC 4. 0.0 - 100.0
+NVML_GPM_METRIC_NVDEC_5_UTIL                = 35 # Percent utilization of NVDEC 5. 0.0 - 100.0
+NVML_GPM_METRIC_NVDEC_6_UTIL                = 36 # Percent utilization of NVDEC 6. 0.0 - 100.0
+NVML_GPM_METRIC_NVDEC_7_UTIL                = 37 # Percent utilization of NVDEC 7. 0.0 - 100.0
+NVML_GPM_METRIC_NVJPG_0_UTIL                = 40 # Percent utilization of NVJPG 0. 0.0 - 100.0
+NVML_GPM_METRIC_NVJPG_1_UTIL                = 41 # Percent utilization of NVJPG 1. 0.0 - 100.0
+NVML_GPM_METRIC_NVJPG_2_UTIL                = 42 # Percent utilization of NVJPG 2. 0.0 - 100.0
+NVML_GPM_METRIC_NVJPG_3_UTIL                = 43 # Percent utilization of NVJPG 3. 0.0 - 100.0
+NVML_GPM_METRIC_NVJPG_4_UTIL                = 44 # Percent utilization of NVJPG 4. 0.0 - 100.0
+NVML_GPM_METRIC_NVJPG_5_UTIL                = 45 # Percent utilization of NVJPG 5. 0.0 - 100.0
+NVML_GPM_METRIC_NVJPG_6_UTIL                = 46 # Percent utilization of NVJPG 6. 0.0 - 100.0
+NVML_GPM_METRIC_NVJPG_7_UTIL                = 47 # Percent utilization of NVJPG 7. 0.0 - 100.0
+NVML_GPM_METRIC_NVOFA_0_UTIL                = 50 # Percent utilization of NVOFA 0. 0.0 - 100.0
+NVML_GPM_METRIC_NVOFA_1_UTIL                = 51 # Percent utilization of NVOFA 1. 0.0 - 100.0
+NVML_GPM_METRIC_NVLINK_TOTAL_RX_PER_SEC     = 60 # NvLink read bandwidth for all links in MiB/sec
+NVML_GPM_METRIC_NVLINK_TOTAL_TX_PER_SEC     = 61 # NvLink write bandwidth for all links in MiB/sec
+NVML_GPM_METRIC_NVLINK_L0_RX_PER_SEC        = 62 # NvLink read bandwidth for link 0 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L0_TX_PER_SEC        = 63 # NvLink write bandwidth for link 0 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L1_RX_PER_SEC        = 64 # NvLink read bandwidth for link 1 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L1_TX_PER_SEC        = 65 # NvLink write bandwidth for link 1 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L2_RX_PER_SEC        = 66 # NvLink read bandwidth for link 2 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L2_TX_PER_SEC        = 67 # NvLink write bandwidth for link 2 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L3_RX_PER_SEC        = 68 # NvLink read bandwidth for link 3 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L3_TX_PER_SEC        = 69 # NvLink write bandwidth for link 3 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L4_RX_PER_SEC        = 70 # NvLink read bandwidth for link 4 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L4_TX_PER_SEC        = 71 # NvLink write bandwidth for link 4 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L5_RX_PER_SEC        = 72 # NvLink read bandwidth for link 5 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L5_TX_PER_SEC        = 73 # NvLink write bandwidth for link 5 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L6_RX_PER_SEC        = 74 # NvLink read bandwidth for link 6 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L6_TX_PER_SEC        = 75 # NvLink write bandwidth for link 6 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L7_RX_PER_SEC        = 76 # NvLink read bandwidth for link 7 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L7_TX_PER_SEC        = 77 # NvLink write bandwidth for link 7 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L8_RX_PER_SEC        = 78 # NvLink read bandwidth for link 8 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L8_TX_PER_SEC        = 79 # NvLink write bandwidth for link 8 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L9_RX_PER_SEC        = 80 # NvLink read bandwidth for link 9 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L9_TX_PER_SEC        = 81 # NvLink write bandwidth for link 9 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L10_RX_PER_SEC       = 82 # NvLink read bandwidth for link 10 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L10_TX_PER_SEC       = 83 # NvLink write bandwidth for link 10 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L11_RX_PER_SEC       = 84 # NvLink read bandwidth for link 11 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L11_TX_PER_SEC       = 85 # NvLink write bandwidth for link 11 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L12_RX_PER_SEC       = 86 # NvLink read bandwidth for link 12 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L12_TX_PER_SEC       = 87 # NvLink write bandwidth for link 12 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L13_RX_PER_SEC       = 88 # NvLink read bandwidth for link 13 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L13_TX_PER_SEC       = 89 # NvLink write bandwidth for link 13 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L14_RX_PER_SEC       = 90 # NvLink read bandwidth for link 14 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L14_TX_PER_SEC       = 91 # NvLink write bandwidth for link 14 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L15_RX_PER_SEC       = 92 # NvLink read bandwidth for link 15 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L15_TX_PER_SEC       = 93 # NvLink write bandwidth for link 15 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L16_RX_PER_SEC       = 94 # NvLink read bandwidth for link 16 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L16_TX_PER_SEC       = 95 # NvLink write bandwidth for link 16 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L17_RX_PER_SEC       = 96 # NvLink read bandwidth for link 17 in MiB/sec
+NVML_GPM_METRIC_NVLINK_L17_TX_PER_SEC       = 97 # NvLink write bandwidth for link 17 in MiB/sec
+NVML_GPM_METRIC_C2C_TOTAL_TX_PER_SEC        = 100
+NVML_GPM_METRIC_C2C_TOTAL_RX_PER_SEC        = 101
+NVML_GPM_METRIC_C2C_DATA_TX_PER_SEC         = 102
+NVML_GPM_METRIC_C2C_DATA_RX_PER_SEC         = 103
+NVML_GPM_METRIC_C2C_LINK0_TOTAL_TX_PER_SEC  = 104
+NVML_GPM_METRIC_C2C_LINK0_TOTAL_RX_PER_SEC  = 105
+NVML_GPM_METRIC_C2C_LINK0_DATA_TX_PER_SEC   = 106
+NVML_GPM_METRIC_C2C_LINK0_DATA_RX_PER_SEC   = 107
+NVML_GPM_METRIC_C2C_LINK1_TOTAL_TX_PER_SEC  = 108
+NVML_GPM_METRIC_C2C_LINK1_TOTAL_RX_PER_SEC  = 109
+NVML_GPM_METRIC_C2C_LINK1_DATA_TX_PER_SEC   = 110
+NVML_GPM_METRIC_C2C_LINK1_DATA_RX_PER_SEC   = 111
+NVML_GPM_METRIC_C2C_LINK2_TOTAL_TX_PER_SEC  = 112
+NVML_GPM_METRIC_C2C_LINK2_TOTAL_RX_PER_SEC  = 113
+NVML_GPM_METRIC_C2C_LINK2_DATA_TX_PER_SEC   = 114
+NVML_GPM_METRIC_C2C_LINK2_DATA_RX_PER_SEC   = 115
+NVML_GPM_METRIC_C2C_LINK3_TOTAL_TX_PER_SEC  = 116
+NVML_GPM_METRIC_C2C_LINK3_TOTAL_RX_PER_SEC  = 117
+NVML_GPM_METRIC_C2C_LINK3_DATA_TX_PER_SEC   = 118
+NVML_GPM_METRIC_C2C_LINK3_DATA_RX_PER_SEC   = 119
+NVML_GPM_METRIC_C2C_LINK4_TOTAL_TX_PER_SEC  = 120
+NVML_GPM_METRIC_C2C_LINK4_TOTAL_RX_PER_SEC  = 121
+NVML_GPM_METRIC_C2C_LINK4_DATA_TX_PER_SEC   = 122
+NVML_GPM_METRIC_C2C_LINK4_DATA_RX_PER_SEC   = 123
+NVML_GPM_METRIC_C2C_LINK5_TOTAL_TX_PER_SEC  = 124
+NVML_GPM_METRIC_C2C_LINK5_TOTAL_RX_PER_SEC  = 125
+NVML_GPM_METRIC_C2C_LINK5_DATA_TX_PER_SEC   = 126
+NVML_GPM_METRIC_C2C_LINK5_DATA_RX_PER_SEC   = 127
+NVML_GPM_METRIC_C2C_LINK6_TOTAL_TX_PER_SEC  = 128
+NVML_GPM_METRIC_C2C_LINK6_TOTAL_RX_PER_SEC  = 129
+NVML_GPM_METRIC_C2C_LINK6_DATA_TX_PER_SEC   = 130
+NVML_GPM_METRIC_C2C_LINK6_DATA_RX_PER_SEC   = 131
+NVML_GPM_METRIC_C2C_LINK7_TOTAL_TX_PER_SEC  = 132
+NVML_GPM_METRIC_C2C_LINK7_TOTAL_RX_PER_SEC  = 133
+NVML_GPM_METRIC_C2C_LINK7_DATA_TX_PER_SEC   = 134
+NVML_GPM_METRIC_C2C_LINK7_DATA_RX_PER_SEC   = 135
+NVML_GPM_METRIC_C2C_LINK8_TOTAL_TX_PER_SEC  = 136
+NVML_GPM_METRIC_C2C_LINK8_TOTAL_RX_PER_SEC  = 137
+NVML_GPM_METRIC_C2C_LINK8_DATA_TX_PER_SEC   = 138
+NVML_GPM_METRIC_C2C_LINK8_DATA_RX_PER_SEC   = 139
+NVML_GPM_METRIC_C2C_LINK9_TOTAL_TX_PER_SEC  = 140
+NVML_GPM_METRIC_C2C_LINK9_TOTAL_RX_PER_SEC  = 141
+NVML_GPM_METRIC_C2C_LINK9_DATA_TX_PER_SEC   = 142
+NVML_GPM_METRIC_C2C_LINK9_DATA_RX_PER_SEC   = 143
+NVML_GPM_METRIC_C2C_LINK10_TOTAL_TX_PER_SEC = 144
+NVML_GPM_METRIC_C2C_LINK10_TOTAL_RX_PER_SEC = 145
+NVML_GPM_METRIC_C2C_LINK10_DATA_TX_PER_SEC  = 146
+NVML_GPM_METRIC_C2C_LINK10_DATA_RX_PER_SEC  = 147
+NVML_GPM_METRIC_C2C_LINK11_TOTAL_TX_PER_SEC = 148
+NVML_GPM_METRIC_C2C_LINK11_TOTAL_RX_PER_SEC = 149
+NVML_GPM_METRIC_C2C_LINK11_DATA_TX_PER_SEC  = 150
+NVML_GPM_METRIC_C2C_LINK11_DATA_RX_PER_SEC  = 151
+NVML_GPM_METRIC_C2C_LINK12_TOTAL_TX_PER_SEC = 152
+NVML_GPM_METRIC_C2C_LINK12_TOTAL_RX_PER_SEC = 153
+NVML_GPM_METRIC_C2C_LINK12_DATA_TX_PER_SEC  = 154
+NVML_GPM_METRIC_C2C_LINK12_DATA_RX_PER_SEC  = 155
+NVML_GPM_METRIC_C2C_LINK13_TOTAL_TX_PER_SEC = 156
+NVML_GPM_METRIC_C2C_LINK13_TOTAL_RX_PER_SEC = 157
+NVML_GPM_METRIC_C2C_LINK13_DATA_TX_PER_SEC  = 158
+NVML_GPM_METRIC_C2C_LINK13_DATA_RX_PER_SEC  = 159
+NVML_GPM_METRIC_HOSTMEM_CACHE_HIT           = 160
+NVML_GPM_METRIC_HOSTMEM_CACHE_MISS          = 161
+NVML_GPM_METRIC_PEERMEM_CACHE_HIT           = 162
+NVML_GPM_METRIC_PEERMEM_CACHE_MISS          = 163
+NVML_GPM_METRIC_DRAM_CACHE_HIT              = 164
+NVML_GPM_METRIC_DRAM_CACHE_MISS             = 165
+NVML_GPM_METRIC_NVENC_0_UTIL                = 166,
+NVML_GPM_METRIC_NVENC_1_UTIL                = 167,
+NVML_GPM_METRIC_NVENC_2_UTIL                = 168,
+NVML_GPM_METRIC_NVENC_3_UTIL                = 169,
+NVML_GPM_METRIC_GR0_CTXSW_CYCLES_ELAPSED    = 170,
+NVML_GPM_METRIC_GR0_CTXSW_CYCLES_ACTIVE     = 171,
+NVML_GPM_METRIC_GR0_CTXSW_REQUESTS          = 172,
+NVML_GPM_METRIC_GR0_CTXSW_CYCLES_PER_REQ    = 173,
+NVML_GPM_METRIC_GR0_CTXSW_ACTIVE_PCT        = 174,
+NVML_GPM_METRIC_GR1_CTXSW_CYCLES_ELAPSED    = 175,
+NVML_GPM_METRIC_GR1_CTXSW_CYCLES_ACTIVE     = 176,
+NVML_GPM_METRIC_GR1_CTXSW_REQUESTS          = 177,
+NVML_GPM_METRIC_GR1_CTXSW_CYCLES_PER_REQ    = 178,
+NVML_GPM_METRIC_GR1_CTXSW_ACTIVE_PCT        = 179,
+NVML_GPM_METRIC_GR2_CTXSW_CYCLES_ELAPSED    = 180,
+NVML_GPM_METRIC_GR2_CTXSW_CYCLES_ACTIVE     = 181,
+NVML_GPM_METRIC_GR2_CTXSW_REQUESTS          = 182,
+NVML_GPM_METRIC_GR2_CTXSW_CYCLES_PER_REQ    = 183,
+NVML_GPM_METRIC_GR2_CTXSW_ACTIVE_PCT        = 184,
+NVML_GPM_METRIC_GR3_CTXSW_CYCLES_ELAPSED    = 185,
+NVML_GPM_METRIC_GR3_CTXSW_CYCLES_ACTIVE     = 186,
+NVML_GPM_METRIC_GR3_CTXSW_REQUESTS          = 187,
+NVML_GPM_METRIC_GR3_CTXSW_CYCLES_PER_REQ    = 188,
+NVML_GPM_METRIC_GR3_CTXSW_ACTIVE_PCT        = 189,
+NVML_GPM_METRIC_GR4_CTXSW_CYCLES_ELAPSED    = 190,
+NVML_GPM_METRIC_GR4_CTXSW_CYCLES_ACTIVE     = 191,
+NVML_GPM_METRIC_GR4_CTXSW_REQUESTS          = 192,
+NVML_GPM_METRIC_GR4_CTXSW_CYCLES_PER_REQ    = 193,
+NVML_GPM_METRIC_GR4_CTXSW_ACTIVE_PCT        = 194,
+NVML_GPM_METRIC_GR5_CTXSW_CYCLES_ELAPSED    = 195,
+NVML_GPM_METRIC_GR5_CTXSW_CYCLES_ACTIVE     = 196,
+NVML_GPM_METRIC_GR5_CTXSW_REQUESTS          = 197,
+NVML_GPM_METRIC_GR5_CTXSW_CYCLES_PER_REQ    = 198,
+NVML_GPM_METRIC_GR5_CTXSW_ACTIVE_PCT        = 199,
+NVML_GPM_METRIC_GR6_CTXSW_CYCLES_ELAPSED    = 200,
+NVML_GPM_METRIC_GR6_CTXSW_CYCLES_ACTIVE     = 201,
+NVML_GPM_METRIC_GR6_CTXSW_REQUESTS          = 202,
+NVML_GPM_METRIC_GR6_CTXSW_CYCLES_PER_REQ    = 203,
+NVML_GPM_METRIC_GR6_CTXSW_ACTIVE_PCT        = 204,
+NVML_GPM_METRIC_GR7_CTXSW_CYCLES_ELAPSED    = 205,
+NVML_GPM_METRIC_GR7_CTXSW_CYCLES_ACTIVE     = 206,
+NVML_GPM_METRIC_GR7_CTXSW_REQUESTS          = 207,
+NVML_GPM_METRIC_GR7_CTXSW_CYCLES_PER_REQ    = 208,
+NVML_GPM_METRIC_GR7_CTXSW_ACTIVE_PCT        = 209,
+NVML_GPM_METRIC_MAX                         = 210
+
+## Structs
+
+class c_nvmlUnitInfo_t(_PrintableStructure):
+    _fields_ = [
+        ('name', c_char * 96),
+        ('id', c_char * 96),
+        ('serial', c_char * 96),
+        ('firmwareVersion', c_char * 96),
+    ]
+
+class struct_c_nvmlGpmSample_t(Structure):
+    pass # opaque handle
+c_nvmlGpmSample_t = POINTER(struct_c_nvmlGpmSample_t)
+
+class c_metricInfo_t(Structure):
+    _fields_ = [
+        ("shortName", c_char_p),
+        ("longName", c_char_p),
+        ("unit", c_char_p),
+    ]
+
+class c_nvmlGpmMetric_t(_PrintableStructure):
+    _fields_ = [
+        ('metricId', c_uint),
+        ('nvmlReturn', _nvmlReturn_t),
+        ('value', c_double),
+        ('metricInfo', c_metricInfo_t)
+    ]
+
+class c_nvmlGpmMetricsGet_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('numMetrics', c_uint),
+        ('sample1', c_nvmlGpmSample_t),
+        ('sample2', c_nvmlGpmSample_t),
+        ('metrics', c_nvmlGpmMetric_t * NVML_GPM_METRIC_MAX)
+    ]
+
+NVML_GPM_METRICS_GET_VERSION = 1
+
+class c_nvmlGpmSupport_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('isSupportedDevice', c_uint),
+    ]
+
+NVML_GPM_SUPPORT_VERSION = 1
+
+## Functions
+
+def nvmlGpmMetricsGet(metricsGet):
+    fn = _nvmlGetFunctionPointer("nvmlGpmMetricsGet")
+    ret = fn(byref(metricsGet))
+    _nvmlCheckReturn(ret)
+    return metricsGet
+
+def nvmlGpmSampleFree(gpmSample):
+    fn = _nvmlGetFunctionPointer("nvmlGpmSampleFree")
+    ret = fn(gpmSample)
+    _nvmlCheckReturn(ret)
+    return
+
+def nvmlGpmSampleAlloc():
+    gpmSample = c_nvmlGpmSample_t()
+    fn = _nvmlGetFunctionPointer("nvmlGpmSampleAlloc")
+    ret = fn(byref(gpmSample))
+    _nvmlCheckReturn(ret)
+    return gpmSample
+
+def nvmlGpmSampleGet(device, gpmSample):
+    fn = _nvmlGetFunctionPointer("nvmlGpmSampleGet")
+    ret = fn(device, gpmSample)
+    _nvmlCheckReturn(ret)
+    return gpmSample
+
+def nvmlGpmMigSampleGet(device, gpuInstanceId, gpmSample):
+    fn = _nvmlGetFunctionPointer("nvmlGpmMigSampleGet")
+    ret = fn(device, gpuInstanceId, gpmSample)
+    _nvmlCheckReturn(ret)
+    return gpmSample
+
+def nvmlGpmQueryDeviceSupport(device):
+    gpmSupport = c_nvmlGpmSupport_t()
+    gpmSupport.version = NVML_GPM_SUPPORT_VERSION
+    fn = _nvmlGetFunctionPointer("nvmlGpmQueryDeviceSupport")
+    ret = fn(device, byref(gpmSupport))
+    _nvmlCheckReturn(ret)
+    return gpmSupport
+
+def nvmlGpmSetStreamingEnabled(device, state):
+    c_state = c_uint(state)
+    fn = _nvmlGetFunctionPointer("nvmlGpmSetStreamingEnabled")
+    ret = fn(device, c_state)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlGpmQueryIfStreamingEnabled(device):
+    c_state = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlGpmQueryIfStreamingEnabled")
+    ret = fn(device, byref(c_state))
+    _nvmlCheckReturn(ret)
+    return c_state.value
+
+# Low Power Structure and Function
+
+NVML_NVLINK_POWER_STATE_HIGH_SPEED    = 0x0
+NVML_NVLINK_POWER_STATE_LOW           = 0x1
+
+NVML_NVLINK_LOW_POWER_THRESHOLD_MIN     = 0x1
+NVML_NVLINK_LOW_POWER_THRESHOLD_MAX     = 0x1FFF
+NVML_NVLINK_LOW_POWER_THRESHOLD_RESET   = 0xFFFFFFFF
+NVML_NVLINK_LOW_POWER_THRESHOLD_DEFAULT = NVML_NVLINK_LOW_POWER_THRESHOLD_RESET
+
+class c_nvmlNvLinkPowerThres_t(Structure):
+    _fields_ = [
+        ("lowPwrThreshold", c_uint),
+    ]
+
+def nvmlDeviceSetNvLinkDeviceLowPowerThreshold(device, l1threshold):
+    c_info = c_nvmlNvLinkPowerThres_t()
+    c_info.lowPwrThreshold = l1threshold
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetNvLinkDeviceLowPowerThreshold")
+    ret = fn(device, byref(c_info))
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+NVML_GPU_FABRIC_UUID_LEN = 16
+
+_nvmlGpuFabricState_t = c_uint
+NVML_GPU_FABRIC_STATE_NOT_SUPPORTED = 0
+NVML_GPU_FABRIC_STATE_NOT_STARTED   = 1
+NVML_GPU_FABRIC_STATE_IN_PROGRESS   = 2
+NVML_GPU_FABRIC_STATE_COMPLETED     = 3
+
+class c_nvmlGpuFabricInfo_t(_PrintableStructure):
+    _fields_ = [
+        ("clusterUuid", c_char * NVML_DEVICE_UUID_BUFFER_SIZE),
+        ("status", _nvmlReturn_t),
+        ("cliqueId", c_uint32),
+        ("state", _nvmlGpuFabricState_t)
+    ]
+
+NVML_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_NOT_SUPPORTED = 0
+NVML_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_TRUE          = 1
+NVML_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_FALSE         = 2
+NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_DEGRADED_BW         = 0
+NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_DEGRADED_BW         = 0x11
+
+NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_NOT_SUPPORTED   = 0
+NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_TRUE            = 1
+NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_FALSE           = 2
+NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_RECOVERY           = 2
+NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_RECOVERY           = 0x11
+
+NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_NOT_SUPPORTED  = 0
+NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_TRUE           = 1
+NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_FALSE          = 2
+NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_UNHEALTHY          = 4
+NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_UNHEALTHY          = 0x11
+
+NVML_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_NOT_SUPPORTED = 0
+NVML_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_TRUE          = 1
+NVML_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_FALSE         = 2
+NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ACCESS_TIMEOUT_RECOVERY         = 6
+NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ACCESS_TIMEOUT_RECOVERY         = 0x11
+
+nvmlGpuFabricInfo_v2 = 0x02000024
+
+class c_nvmlGpuFabricInfoV_t(_PrintableStructure):
+    _fields_ = [
+        ("version", c_uint),
+        ("clusterUuid", c_char * NVML_GPU_FABRIC_UUID_LEN),
+        ("status", _nvmlReturn_t),
+        ("cliqueId", c_uint32),
+        ("state", _nvmlGpuFabricState_t),
+        ("healthMask", c_uint32)
+    ]
+
+    def __init__(self):
+        super(c_nvmlGpuFabricInfoV_t, self).__init__(version=nvmlGpuFabricInfo_v2)
+
+def nvmlDeviceGetGpuFabricInfo(device, gpuFabricInfo):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuFabricInfo");
+    ret = fn(device, gpuFabricInfo)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceGetGpuFabricInfoV(device, gpuFabricInfo):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuFabricInfoV");
+    ret = fn(device, gpuFabricInfo)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+######################
+## Enums/defines
+#### NVML GPU NVLINK BW MODE
+NVML_GPU_NVLINK_BW_MODE_FULL      = 0x0
+NVML_GPU_NVLINK_BW_MODE_OFF       = 0x1
+NVML_GPU_NVLINK_BW_MODE_MIN       = 0x2
+NVML_GPU_NVLINK_BW_MODE_HALF      = 0x3
+NVML_GPU_NVLINK_BW_MODE_3QUARTER  = 0x4
+NVML_GPU_NVLINK_BW_MODE_COUNT     = 0x5
+
+def nvmlSystemSetNvlinkBwMode(mode):
+    fn = _nvmlGetFunctionPointer("nvmlSystemSetNvlinkBwMode")
+    ret = fn(mode)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlSystemGetNvlinkBwMode():
+    mode = c_uint()
+    fn = _nvmlGetFunctionPointer("nvmlSystemGetNvlinkBwMode")
+    ret = fn(byref(mode))
+    _nvmlCheckReturn(ret)
+    return mode.value
+
+_nvmlPowerScopeType_t = c_uint
+NVML_POWER_SCOPE_GPU     = 0
+NVML_POWER_SCOPE_MODULE  = 1
+NVML_POWER_SCOPE_MEMORY  = 2
+
+class c_nvmlPowerValue_v2_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('powerScope', _nvmlPowerScopeType_t),
+        ('powerValueMw', c_uint),
+    ]
+    _fmt_ = {'': "%d B"}
+
+nvmlPowerValue_v2 = 0x0200000C
+
+def nvmlDeviceSetPowerManagementLimit_v2(device, powerScope, powerLimit, version=nvmlPowerValue_v2):
+    c_powerScope = _nvmlPowerScopeType_t(powerScope)
+    c_powerValue = c_nvmlPowerValue_v2_t()
+    c_powerValue.version = c_uint(version)
+    c_powerValue.powerScope = c_powerScope
+    c_powerValue.powerValueMw = c_uint(powerLimit)
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetPowerManagementLimit_v2")
+    ret = fn(device, byref(c_powerValue))
+    return NVML_SUCCESS
+
+class c_nvmlEccSramErrorStatus_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('aggregateUncParity', c_ulonglong),
+        ('aggregateUncSecDed', c_ulonglong),
+        ('aggregateCor', c_ulonglong),
+        ('volatileUncParity', c_ulonglong),
+        ('volatileUncSecDed', c_ulonglong),
+        ('volatileCor', c_ulonglong),
+        ('aggregateUncBucketL2', c_ulonglong),
+        ('aggregateUncBucketSm', c_ulonglong),
+        ('aggregateUncBucketPcie', c_ulonglong),
+        ('aggregateUncBucketMcu', c_ulonglong),
+        ('aggregateUncBucketOther', c_ulonglong),
+        ('bThresholdExceeded', c_uint)
+    ]
+
+    def __init__(self):
+        super(c_nvmlEccSramErrorStatus_v1_t, self).__init__(version=nvmlEccSramErrorStatus_v1)
+
+nvmlEccSramErrorStatus_v1 = 0x1000068
+def nvmlDeviceGetSramEccErrorStatus(device, status):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetSramEccErrorStatus")
+    ret = fn(device, status)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+NVML_DEV_CAP_EGM = (1 << 0)
+nvmlDeviceCapabilities_v1 = 0x1000008
+
+class c_nvmlDeviceCapabilities_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('capMask', c_uint),
+    ]
+
+    def __init__(self):
+        super(c_nvmlDeviceCapabilities_v1_t, self).__init__(version=nvmlDeviceCapabilities_v1)
+
+
+def nvmlDeviceGetCapabilities(device, caps):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetCapabilities")
+    return fn(device, caps)
+
+class c_nvmlPlatformInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('ibGuid', c_char * 16),
+        ('rackGuid', c_char * 16),
+        ('chassisPhysicalSlotNumber', c_char),
+        ('computeSlotIndex', c_char),
+        ('nodeIndex', c_char),
+        ('peerType', c_char),
+        ('moduleId', c_char)
+    ]
+
+    def __init__(self):
+        super(c_nvmlPlatformInfo_v1_t, self).__init__(version=nvmlPlatformInfo_v1)
+
+class c_nvmlPlatformInfo_v2_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('ibGuid', c_char * 16),
+        ('chassisSerialNumber', c_char * 16),
+        ('slotNumber', c_char),
+        ('trayIndex', c_char),
+        ('hostId', c_char),
+        ('peerType', c_char),
+        ('moduleId', c_char)
+    ]
+
+    def __init__(self):
+        super(c_nvmlPlatformInfo_v2_t, self).__init__(version=nvmlPlatformInfo_v2)
+
+nvmlPlatformInfo_v1 = 0x100002c
+nvmlPlatformInfo_v2 = 0x200002c
+
+def nvmlDeviceGetPlatformInfo(device, platformInfo):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetPlatformInfo")
+    ret = fn(device, platformInfo)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+class c_nvmlMask255_t(_PrintableStructure):
+    _fields_ = [
+        ('mask', c_uint * 8),
+    ]
+
+NVML_WORKLOAD_POWER_MAX_PROFILES    = 255
+NVML_POWER_PROFILE_MAX_P            = 0
+NVML_POWER_PROFILE_MAX_Q            = 1
+NVML_POWER_PROFILE_COMPUTE          = 2
+NVML_POWER_PROFILE_MEMORY_BOUND     = 3
+NVML_POWER_PROFILE_NETWORK          = 4
+NVML_POWER_PROFILE_BALANCED         = 5
+NVML_POWER_PROFILE_LLM_INFERENCE    = 6
+NVML_POWER_PROFILE_LLM_TRAINING     = 7
+NVML_POWER_PROFILE_RBM              = 8
+NVML_POWER_PROFILE_DCPCIE           = 9
+NVML_POWER_PROFILE_HMMA_SPARSE      = 10
+NVML_POWER_PROFILE_HMMA_DENSE       = 11
+NVML_POWER_PROFILE_SYNC_BALANCED    = 12
+NVML_POWER_PROFILE_HPC              = 13
+NVML_POWER_PROFILE_MIG              = 14
+NVML_POWER_PROFILE_MAX              = 15
+
+nvmlWorkloadPowerProfileInfo_v1 = 0x100002c
+class c_nvmlWorkloadPowerProfileInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('profileId', c_uint),
+        ('priority', c_uint),
+        ('conflictingmask', c_nvmlMask255_t)
+    ]
+
+    def __init__(self):
+        super(c_nvmlWorkloadPowerProfileInfo_v1_t, self).__init__(version=nvmlWorkloadPowerProfileInfo_v1)
+
+nvmlWorkloadPowerProfileProfilesInfo_v1 = 0x1002bf8
+class c_nvmlWorkloadPowerProfileProfilesInfo_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('perfProfilesMask', c_nvmlMask255_t),
+        ('perfProfile', c_nvmlWorkloadPowerProfileInfo_v1_t * NVML_WORKLOAD_POWER_MAX_PROFILES)
+    ]
+
+    def __init__(self):
+        super(c_nvmlWorkloadPowerProfileProfilesInfo_v1_t, self).__init__(version=nvmlWorkloadPowerProfileProfilesInfo_v1)
+
+nvmlWorkloadPowerProfileCurrentProfiles_v1 = 0x1000064
+class c_nvmlWorkloadPowerProfileCurrentProfiles_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('perfProfilesMask', c_nvmlMask255_t),
+        ('requestedProfilesMask', c_nvmlMask255_t),
+        ('enforcedProfilesMask', c_nvmlMask255_t)
+    ]
+
+    def __init__(self):
+        super(c_nvmlWorkloadPowerProfileCurrentProfiles_v1_t, self).__init__(version=nvmlWorkloadPowerProfileCurrentProfiles_v1)
+
+nvmlWorkloadPowerProfileRequestedProfiles_v1 = 0x1000024
+class c_nvmlWorkloadPowerProfileRequestedProfiles_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('requestedProfilesMask', c_nvmlMask255_t),
+    ]
+
+    def __init__(self):
+        super(c_nvmlWorkloadPowerProfileRequestedProfiles_v1_t, self).__init__(version=nvmlWorkloadPowerProfileRequestedProfiles_v1)
+
+def nvmlDeviceWorkloadPowerProfileGetProfilesInfo(device, profilesInfo):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceWorkloadPowerProfileGetProfilesInfo")
+    ret = fn(device, profilesInfo)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceWorkloadPowerProfileGetCurrentProfiles(device, currentProfiles):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceWorkloadPowerProfileGetCurrentProfiles")
+    ret = fn(device, currentProfiles)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceWorkloadPowerProfileSetRequestedProfiles(device, requestedProfiles):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceWorkloadPowerProfileSetRequestedProfiles")
+    ret = fn(device, requestedProfiles)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceWorkloadPowerProfileClearRequestedProfiles(device, requestedProfiles):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceWorkloadPowerProfileClearRequestedProfiles")
+    ret = fn(device, requestedProfiles)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceGetNvlinkSupportedBwModes(device, supportedBwModes):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvlinkSupportedBwModes")
+    ret = fn(device, supportedBwModes)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceGetNvlinkBwMode(device, getBwMode):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvlinkBwMode")
+    ret = fn(device, getBwMode)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+def nvmlDeviceSetNvlinkBwMode(device, setBwMode):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetNvlinkBwMode")
+    ret = fn(device, setBwMode)
+    _nvmlCheckReturn(ret)
+    return NVML_SUCCESS
+
+nvmlDramEncryptionInfo_v1 = 0x01000008
+
+class c_nvmlDramEncryptionInfo_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('encryptionState',  _nvmlEnableState_t),
+    ]
+
+    def __init__(self):
+        super(c_nvmlDramEncryptionInfo_t, self).__init__(version=nvmlDramEncryptionInfo_v1)
+
+def nvmlDeviceGetDramEncryptionMode(handle):
+    c_currState = c_nvmlDramEncryptionInfo_t()
+    c_pendingState = c_nvmlDramEncryptionInfo_t()
+    fn = _nvmlGetFunctionPointer("nvmlDeviceGetDramEncryptionMode")
+    ret = fn(handle, byref(c_currState), byref(c_pendingState))
+    _nvmlCheckReturn(ret)
+    return [c_currState.encryptionState, c_pendingState.encryptionState]
+
+# added to API
+def nvmlDeviceGetCurrentDramEncryptionMode(handle):
+    return nvmlDeviceGetDramEncryptionMode(handle)[0]
+
+# added to API
+def nvmlDeviceGetPendingDramEncryptionMode(handle):
+    return nvmlDeviceGetDramEncryptionMode(handle)[1]
+
+def nvmlDeviceSetDramEncryptionMode(handle, mode):
+    fn = _nvmlGetFunctionPointer("nvmlDeviceSetDramEncryptionMode")
+    c_dramEncryptionMode = c_nvmlDramEncryptionInfo_t()
+    c_dramEncryptionMode.encryptionState = mode;
+    ret = fn(handle, byref(c_dramEncryptionMode))
+    _nvmlCheckReturn(ret)
+    return None
+
+# Power Smoothing defines
+NVML_POWER_SMOOTHING_MAX_NUM_PROFILES                   = 5
+NVML_POWER_SMOOTHING_ADMIN_OVERRIDE_NOT_SET             = 0xFFFFFFFF
+NVML_POWER_SMOOTHING_PROFILE_PARAM_PERCENT_TMP_FLOOR    = 0
+NVML_POWER_SMOOTHING_PROFILE_PARAM_RAMP_UP_RATE         = 1
+NVML_POWER_SMOOTHING_PROFILE_PARAM_RAMP_DOWN_RATE       = 2
+NVML_POWER_SMOOTHING_PROFILE_PARAM_RAMP_DOWN_HYSTERESIS = 3
+
+nvmlPowerSmoothingState_v1=0x1000008
+class c_nvmlPowerSmoothingState_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('state', c_uint),
+    ]
+
+    def __init__(self):
+        super(c_nvmlPowerSmoothingState_v1_t, self).__init__(version=nvmlPowerSmoothingState_v1)
+
+nvmlPowerSmoothingProfile_v1=0x1000018
+class c_nvmlPowerSmoothingProfile_v1_t(_PrintableStructure):
+    _fields_ = [
+        ('version', c_uint),
+        ('profileId', c_uint),
+        ('paramId', c_uint),
+        ('value', c_double),
+    ]
+
+    def __init__(self):
+        super(c_nvmlPowerSmoothingProfile_v1_t, self).__init__(version=nvmlPowerSmoothingProfile_v1)
+
+def nvmlDevicePowerSmoothingActivatePresetProfile(device, profile):
+    fn = _nvmlGetFunctionPointer("nvmlDevicePowerSmoothingActivatePresetProfile")
+    ret = fn(device, profile)
+    _nvmlCheckReturn(ret)
+
+def nvmlDevicePowerSmoothingUpdatePresetProfileParam(device, profile):
+    fn = _nvmlGetFunctionPointer("nvmlDevicePowerSmoothingUpdatePresetProfileParam")
+    ret = fn(device, profile)
+    _nvmlCheckReturn(ret)
+
+def nvmlDevicePowerSmoothingSetState(device, state):
+    fn = _nvmlGetFunctionPointer("nvmlDevicePowerSmoothingSetState")
+    ret = fn(device, state)
+    _nvmlCheckReturn(ret)
+
diff --git a/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/INSTALLER b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/INSTALLER
new file mode 100644
index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68
--- /dev/null
+++ b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/METADATA b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..18f4768e001c66412a76be38d5c071c7b6dfd50a
--- /dev/null
+++ b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/METADATA
@@ -0,0 +1,139 @@
+Metadata-Version: 2.2
+Name: pyzmq
+Version: 27.0.1
+Summary: Python bindings for 0MQ
+Author: Brian E. Granger, Min Ragan-Kelley
+Author-Email: PyZMQ Contributors 
+License: BSD 3-Clause License
+         
+         Copyright (c) 2009-2012, Brian Granger, Min Ragan-Kelley
+         
+         All rights reserved.
+         
+         Redistribution and use in source and binary forms, with or without
+         modification, are permitted provided that the following conditions are met:
+         
+         1. Redistributions of source code must retain the above copyright notice, this
+            list of conditions and the following disclaimer.
+         
+         2. Redistributions in binary form must reproduce the above copyright notice,
+            this list of conditions and the following disclaimer in the documentation
+            and/or other materials provided with the distribution.
+         
+         3. Neither the name of the copyright holder nor the names of its
+            contributors may be used to endorse or promote products derived from
+            this software without specific prior written permission.
+         
+         THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+         AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+         IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+         DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+         FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+         DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+         SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+         CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+         OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+         OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+         
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX
+Classifier: Topic :: System :: Networking
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Project-URL: Homepage, https://pyzmq.readthedocs.org
+Project-URL: Documentation, https://pyzmq.readthedocs.org
+Project-URL: Source, https://github.com/zeromq/pyzmq
+Project-URL: Tracker, https://github.com/zeromq/pyzmq/issues
+Requires-Python: >=3.8
+Requires-Dist: cffi; implementation_name == "pypy"
+Description-Content-Type: text/markdown
+
+# PyZMQ: Python bindings for ØMQ
+
+This package contains Python bindings for [ZeroMQ](https://zeromq.org).
+ØMQ is a lightweight and fast messaging implementation.
+
+PyZMQ should work with any reasonable version of Python (≥ 3.8), as well as PyPy.
+PyZMQ supports libzmq ≥ 3.2.2 (including 4.x).
+
+For a summary of changes to pyzmq, see our
+[changelog](https://pyzmq.readthedocs.io/en/latest/changelog.html).
+
+### ØMQ 3.x, 4.x
+
+PyZMQ fully supports the stable (not DRAFT) 3.x and 4.x APIs of libzmq,
+developed at [zeromq/libzmq](https://github.com/zeromq/libzmq).
+No code to change, no flags to pass,
+just build pyzmq against the latest and it should work.
+
+## Documentation
+
+See PyZMQ's Sphinx-generated
+documentation [on Read the Docs](https://pyzmq.readthedocs.io) for API
+details, and some notes on Python and Cython development. If you want to
+learn about using ØMQ in general, the excellent [ØMQ
+Guide](http://zguide.zeromq.org/py:all) is the place to start, which has a
+Python version of every example. We also have some information on our
+[wiki](https://github.com/zeromq/pyzmq/wiki).
+
+## Downloading
+
+Unless you specifically want to develop PyZMQ, we recommend downloading
+the PyZMQ source code or wheels from
+[PyPI](https://pypi.io/project/pyzmq/),
+or install with conda.
+
+You can also get the latest source code from our GitHub repository, but
+building from the repository will require that you install recent Cython.
+
+## Building and installation
+
+For more detail on building pyzmq, see [our docs](https://pyzmq.readthedocs.io/en/latest/howto/build.html).
+
+We build wheels for macOS, Windows, and Linux, so you can get a binary on those platforms with:
+
+```
+pip install pyzmq
+```
+
+but compiling from source with `pip install pyzmq` should work in most environments.
+Make sure you are using the latest pip, or it may not find the right wheels.
+
+If the wheel doesn't work for some reason, or you want to force pyzmq to be compiled
+(this is often preferable if you already have libzmq installed and configured the way you want it),
+you can force installation from source with:
+
+```
+pip install --no-binary=pyzmq pyzmq
+```
+
+## Old versions
+
+pyzmq 16 drops support Python 2.6 and 3.2.
+If you need to use one of those Python versions, you can pin your pyzmq version to before 16:
+
+```
+pip install 'pyzmq<16'
+```
+
+For libzmq 2.0.x, use 'pyzmq\<2.1'
+
+pyzmq-2.1.11 was the last version of pyzmq to support Python 2.5,
+and pyzmq ≥ 2.2.0 requires Python ≥ 2.6.
+pyzmq-13.0.0 introduces PyPy support via CFFI, which only supports libzmq-3.2.2 and newer.
+
+PyZMQ releases ≤ 2.2.0 matched libzmq versioning, but this is no longer the case,
+starting with PyZMQ 13.0.0 (it was the thirteenth release, so why not?).
+PyZMQ ≥ 13.0 follows semantic versioning conventions accounting only for PyZMQ itself.
diff --git a/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/RECORD b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/RECORD
new file mode 100644
index 0000000000000000000000000000000000000000..1bdaae87285ba7031080614b4cdea5f172faa7d5
--- /dev/null
+++ b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/RECORD
@@ -0,0 +1,164 @@
+pyzmq-27.0.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+pyzmq-27.0.1.dist-info/METADATA,sha256=wjsMSGMm1AWhyy1ZhVgs8FkrKMqU9RZzbGA-U8PWHMI,6008
+pyzmq-27.0.1.dist-info/RECORD,,
+pyzmq-27.0.1.dist-info/WHEEL,sha256=6ecSNHkZ6bSCAmjnCo3X3oeWgXRHh_xwRO605ZPt-HI,157
+pyzmq-27.0.1.dist-info/licenses/LICENSE.md,sha256=wM9fXAP41ncveicd8ctnEFRXi9PXlSfHL8Hyj4zHKno,1545
+pyzmq-27.0.1.dist-info/licenses/licenses/LICENSE.libsodium.txt,sha256=Q5ZNl2pts_uYavaJ0F-MoOmXGHi8yucJdQ2sj9xKmc8,823
+pyzmq-27.0.1.dist-info/licenses/licenses/LICENSE.tornado.txt,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+pyzmq-27.0.1.dist-info/licenses/licenses/LICENSE.zeromq.txt,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725
+pyzmq.libs/libsodium-19479d6d.so.26.2.0,sha256=DdsACXJv1Ophz2bywp5SgprpXMZgDjkqCXSzEeQbG-M,1209897
+pyzmq.libs/libzmq-7b073b3d.so.5.2.5,sha256=gRVz7TrzVPqNpR4KewVc_jjRXqa9BnosbEl0tV3jjIs,950593
+zmq/__init__.pxd,sha256=P2y5B_9nDB_0RD7hxbXpVm4Jia1rKclTnnUVrSbF4lE,63
+zmq/__init__.py,sha256=0zUxdN9mC6mBJAOkBfsI4em8roHbvH0afHQTYsMFjXA,2232
+zmq/__init__.pyi,sha256=2Vk8hzDtjG-PtgybCgyA3B31dxk17IwtzkuXLrauLdc,922
+zmq/__pycache__/__init__.cpython-310.pyc,,
+zmq/__pycache__/_future.cpython-310.pyc,,
+zmq/__pycache__/_typing.cpython-310.pyc,,
+zmq/__pycache__/asyncio.cpython-310.pyc,,
+zmq/__pycache__/constants.cpython-310.pyc,,
+zmq/__pycache__/decorators.cpython-310.pyc,,
+zmq/__pycache__/error.cpython-310.pyc,,
+zmq/_future.py,sha256=jLLA1bP3zC_JV6H1F3ZY7qO_URPaaCuAjb9vQ2e6Df0,24340
+zmq/_future.pyi,sha256=HLN20gfpZRfe9RMpW_kdsmnLrlxlO8r-W-TAuyWvZGM,3322
+zmq/_typing.py,sha256=FArL9c6FqYiEdVjv5Wf5-1nJnbZcD3KIjRm17mOxHrI,241
+zmq/asyncio.py,sha256=NjDmuZakTa2_4tvEH6Kn6j8s-bJ6JO0N7QscPdx_bfo,6518
+zmq/auth/__init__.py,sha256=D0XJjPJgN0ZqSBLDrbLm3l3N5pMQt75pv8LyearhsM8,346
+zmq/auth/__pycache__/__init__.cpython-310.pyc,,
+zmq/auth/__pycache__/asyncio.cpython-310.pyc,,
+zmq/auth/__pycache__/base.cpython-310.pyc,,
+zmq/auth/__pycache__/certs.cpython-310.pyc,,
+zmq/auth/__pycache__/ioloop.cpython-310.pyc,,
+zmq/auth/__pycache__/thread.cpython-310.pyc,,
+zmq/auth/asyncio.py,sha256=KLD0Kwev61dnImVhcLmEKr-PwTCqIyurWjs4SuH442A,1799
+zmq/auth/base.py,sha256=OPTB58nTeYJL-bu9bHa4lXUCCMwzo7cwlhxRuHBjd0Y,16337
+zmq/auth/certs.py,sha256=0lyPqG3o-ucI_UvCVpihxT10V9-hKJcyi5Us4trVGR0,4329
+zmq/auth/ioloop.py,sha256=xXF6P8A-HZlXfIYMVv8BW8EI_H2PjrJFF3a6Ajvd1rI,1298
+zmq/auth/thread.py,sha256=mv1NfTxJIydLhIkKdExPVDdWpcbczUDshIsYLUzhSkI,4103
+zmq/backend/__init__.py,sha256=5PfcIfpIzxGIegOaHbO3dBilBzby3l0yKC0o_Qf1m3A,940
+zmq/backend/__init__.pyi,sha256=yYO_6IRAThgQBnMviaA8ZSKtAb-B2y7wzotSDWT_hqk,3370
+zmq/backend/__pycache__/__init__.cpython-310.pyc,,
+zmq/backend/__pycache__/select.cpython-310.pyc,,
+zmq/backend/cffi/README.md,sha256=u7zNkS3dJALRPzpgPv5s4Q1tIkevm0BMzVpcwZD0PoM,95
+zmq/backend/cffi/__init__.py,sha256=wNVPr6F3_fGxvBjQclPfZizcn59Yhh2Z_20XwNw8JXk,898
+zmq/backend/cffi/__pycache__/__init__.cpython-310.pyc,,
+zmq/backend/cffi/__pycache__/_poll.cpython-310.pyc,,
+zmq/backend/cffi/__pycache__/context.cpython-310.pyc,,
+zmq/backend/cffi/__pycache__/devices.cpython-310.pyc,,
+zmq/backend/cffi/__pycache__/error.cpython-310.pyc,,
+zmq/backend/cffi/__pycache__/message.cpython-310.pyc,,
+zmq/backend/cffi/__pycache__/socket.cpython-310.pyc,,
+zmq/backend/cffi/__pycache__/utils.cpython-310.pyc,,
+zmq/backend/cffi/_cdefs.h,sha256=vidXzyrwaPkZjeDKy1ebyOozC_QncibN_lYLr6F4ujs,2860
+zmq/backend/cffi/_cffi_src.c,sha256=ADxHwdYLRjsRf8vuyOK8lbHiXTjgbfoiIwOjarFQ8ds,1314
+zmq/backend/cffi/_poll.py,sha256=niEuO85_fLbRntELWu0k89qLicJPgnirX13fTnw0Irc,2884
+zmq/backend/cffi/context.py,sha256=dKoVS0VJa0A1N3l9LYGRoYlRLmmHZigmnMhZyHsP-jA,1899
+zmq/backend/cffi/devices.py,sha256=r3sLr3lyrj0mjVx_NenHQAtA9BWyhsUGuJoCHmplLXg,1480
+zmq/backend/cffi/error.py,sha256=h-mZPwc1TydObeTU3XUmMi9KUeqs3NJYNTqUTkt5dyI,311
+zmq/backend/cffi/message.py,sha256=7AOjzkymUuPNNKu1wzY4qw09ex1xpy2Wy9M69km0G3Y,6675
+zmq/backend/cffi/socket.py,sha256=jEARhOFTTaC6mLQbI2-yBxJ19y6bNWVT6NyOqEfFf0g,14217
+zmq/backend/cffi/utils.py,sha256=fN1m75YzUgZCLhrFnlBWiwB-hS3g8ri9mxstMyU7dKY,2056
+zmq/backend/cython/__init__.pxd,sha256=iRgsrNY8-yEX3UL83jFHziSPaVibZx-qltTXcYVUM9Y,60
+zmq/backend/cython/__init__.py,sha256=SMuE8GAtsmEMWNGrv-ZlQZcuTD3EVU838E1ViRHLDNI,322
+zmq/backend/cython/__pycache__/__init__.cpython-310.pyc,,
+zmq/backend/cython/__pycache__/_zmq.cpython-310.pyc,,
+zmq/backend/cython/_externs.pxd,sha256=iVF0FfbYZU1qPYvhmi8IhMKBgVgjSCYWJsgjl0c6FTk,349
+zmq/backend/cython/_zmq.cpython-310-x86_64-linux-gnu.so,sha256=Laa6d_o5s3Q3WyCYYNvGHS18-o8X_tzokzwMdfLk8Rk,291337
+zmq/backend/cython/_zmq.pxd,sha256=jakQv3gt9PnKcYtbw7Cf7XpAYY4rns64iU2AOzCCwuw,2378
+zmq/backend/cython/_zmq.py,sha256=v1XPHepRXz4f4IrZJ8jzimrRhdOHK6WZDHvZUPNXqqo,61871
+zmq/backend/cython/constant_enums.pxi,sha256=LNVbov9C6GBuJvWHnfpqUjmNT0x8alTeub885-o_mI0,7562
+zmq/backend/cython/libzmq.pxd,sha256=QNpBWm5w62Z4sQzIxQz8Y0n5WwFlILRu5ws7E8tVURE,4830
+zmq/backend/select.py,sha256=wpFbaaYbWJn89NTzS9dPyum8JJdNgakEICTFLI7t-Ws,888
+zmq/constants.py,sha256=xEyRW8hA1lLjDAnMjOWvmKAFQqZYzaTWO62dA7atnbM,28341
+zmq/decorators.py,sha256=sLeTjxsNcnjKYCsyUsx5RyC0X2Sfqi355nvBDzLDxGY,5099
+zmq/devices/__init__.py,sha256=Ao9r2eNUSgj3We4Zv0CN54jRKsrVoLf9PXYCQQbLQZI,769
+zmq/devices/__pycache__/__init__.cpython-310.pyc,,
+zmq/devices/__pycache__/basedevice.cpython-310.pyc,,
+zmq/devices/__pycache__/monitoredqueue.cpython-310.pyc,,
+zmq/devices/__pycache__/monitoredqueuedevice.cpython-310.pyc,,
+zmq/devices/__pycache__/proxydevice.cpython-310.pyc,,
+zmq/devices/__pycache__/proxysteerabledevice.cpython-310.pyc,,
+zmq/devices/basedevice.py,sha256=Ma39_n1J6LzTrhTz3ZxoXZaniDksCpzGYPDhlK0nvPg,9542
+zmq/devices/monitoredqueue.py,sha256=au2EN-fFLXDvVRFClAYS3q2Lb-KxW3keT9l82W5BKRo,1294
+zmq/devices/monitoredqueuedevice.py,sha256=onjY-L74VC_YjppiC4C_voBgbuEtRhstdTaytxQe14I,1929
+zmq/devices/proxydevice.py,sha256=MCB4j-65vyjSLytrzEWfB2q6YZLn_035Sxfns9j4yyQ,2843
+zmq/devices/proxysteerabledevice.py,sha256=atHF4HRd7A_lQQV8q6eHzQ_BaT-Gv6D3EabnN1p67vQ,3206
+zmq/error.py,sha256=CRQbIietfHLG2bV6mvBRIIYFNABpuGBbz9s0U9vC3E0,5961
+zmq/eventloop/__init__.py,sha256=j5PpZdLAwLtwChrGCEZHJYJ6ZJoEzNBMlzY9r5K5iUw,103
+zmq/eventloop/__pycache__/__init__.cpython-310.pyc,,
+zmq/eventloop/__pycache__/_deprecated.cpython-310.pyc,,
+zmq/eventloop/__pycache__/future.cpython-310.pyc,,
+zmq/eventloop/__pycache__/ioloop.cpython-310.pyc,,
+zmq/eventloop/__pycache__/zmqstream.cpython-310.pyc,,
+zmq/eventloop/_deprecated.py,sha256=APinUSjjM4B3Bn2VlqXzR4OJj_RkyQbWJUDvjndStsY,6443
+zmq/eventloop/future.py,sha256=lueaaPliVxJkvTaksnmAJkd09XZUfi4o0YnAQiFsciI,2612
+zmq/eventloop/ioloop.py,sha256=pmFSoqjZUy40wbibneTYwyDfVy43a1Ffvzus3pdelU4,766
+zmq/eventloop/zmqstream.py,sha256=VTvYZnxJbT9uwHN2opzYQPcXT59divswpwcvU52NMCE,23041
+zmq/green/__init__.py,sha256=Vmg7Zv4rXt9dUbgy7pGx1a8igWFMqcIWRnRrzZq3Jx4,1367
+zmq/green/__pycache__/__init__.cpython-310.pyc,,
+zmq/green/__pycache__/core.cpython-310.pyc,,
+zmq/green/__pycache__/device.cpython-310.pyc,,
+zmq/green/__pycache__/poll.cpython-310.pyc,,
+zmq/green/core.py,sha256=CdD5t314XkL5ZcAszaaKdg3ceCSZjzeunPrX81_THTI,11447
+zmq/green/device.py,sha256=HTtdyyENo8aOtpklkoDnfSVJKbM5Xh_-KMJZbImZgSQ,978
+zmq/green/eventloop/__init__.py,sha256=N13sRnQlJDo2gD70qPNZP7uc_EEMAjE6hDa-SLhKj0s,68
+zmq/green/eventloop/__pycache__/__init__.cpython-310.pyc,,
+zmq/green/eventloop/__pycache__/ioloop.cpython-310.pyc,,
+zmq/green/eventloop/__pycache__/zmqstream.cpython-310.pyc,,
+zmq/green/eventloop/ioloop.py,sha256=rNJvPZsF-SZpXFEk7T8DXUE5yMFxltF5HE9qZkCmufc,43
+zmq/green/eventloop/zmqstream.py,sha256=3LGGOp9Lx0OxrsiNNxt4jdzNAJvXZNMLdlOYcsrDz8c,291
+zmq/green/poll.py,sha256=77Jpd7h-TJ0ZLE7vm1J7hfJfRCm3BI41cL5CYmUzH1A,2996
+zmq/log/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+zmq/log/__main__.py,sha256=FsHekF9qqnfhDkGvl8zVWxUuckLxTqubxMr6GjuCyTA,4005
+zmq/log/__pycache__/__init__.cpython-310.pyc,,
+zmq/log/__pycache__/__main__.cpython-310.pyc,,
+zmq/log/__pycache__/handlers.cpython-310.pyc,,
+zmq/log/handlers.py,sha256=PLdJvzN3J6pg1Z4mied6RxwoKoW6VF8vz1mivTMl74Y,7228
+zmq/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+zmq/ssh/__init__.py,sha256=2Wcr18a8hS5Qjwhn1p6SYb6NMDIi7Y48JRXg56iU0fI,29
+zmq/ssh/__pycache__/__init__.cpython-310.pyc,,
+zmq/ssh/__pycache__/forward.cpython-310.pyc,,
+zmq/ssh/__pycache__/tunnel.cpython-310.pyc,,
+zmq/ssh/forward.py,sha256=B0tgdTGziNmhbSNUCk-BcrdCDmSUp3ev18tGOz6Hg8U,3246
+zmq/ssh/tunnel.py,sha256=SCurCVb4WkKFZluqDKQzMkTsmlM0SHRNno5fUNRkrUk,13357
+zmq/sugar/__init__.py,sha256=pNZlBhg8DnJPQoodyICE1zlPsABcpzDcipPVE3PTkw4,1061
+zmq/sugar/__init__.pyi,sha256=F_JYIucugCUuXik_FSVfzWXICyuH1yDzlshcZRb8bDU,219
+zmq/sugar/__pycache__/__init__.cpython-310.pyc,,
+zmq/sugar/__pycache__/attrsettr.cpython-310.pyc,,
+zmq/sugar/__pycache__/context.cpython-310.pyc,,
+zmq/sugar/__pycache__/frame.cpython-310.pyc,,
+zmq/sugar/__pycache__/poll.cpython-310.pyc,,
+zmq/sugar/__pycache__/socket.cpython-310.pyc,,
+zmq/sugar/__pycache__/stopwatch.cpython-310.pyc,,
+zmq/sugar/__pycache__/tracker.cpython-310.pyc,,
+zmq/sugar/__pycache__/version.cpython-310.pyc,,
+zmq/sugar/attrsettr.py,sha256=LL3MjUFm2TW4VYTsmv1FwEATM3Qw_MGT6s9fB1mBVek,2638
+zmq/sugar/context.py,sha256=hGIxHjkE7urLIFDhuRXSIR9atKsBPXrkvtIEigrH8KA,14575
+zmq/sugar/frame.py,sha256=OpWj-JQypaq54XDpgfUtVSFTpJ7KPwwd6B562DrJmm8,4264
+zmq/sugar/poll.py,sha256=7qQnUTtQJL8IN5S2VeB0jheoqUKp37pM20yjt4Fv6P4,5752
+zmq/sugar/socket.py,sha256=Lf8ksPBLVBVNiq1it140DURWVt-1VmBlWnUsWWoANwc,35362
+zmq/sugar/stopwatch.py,sha256=i1Cg96aPzsiHmUTAZEgSsiZ5qQJ7rw-pFgiIYJoJU1g,935
+zmq/sugar/tracker.py,sha256=raZKyJc3SYxlY17uAQpIPkHUaNk7bt9cwtdYugLd8QQ,3603
+zmq/sugar/version.py,sha256=Jc3vpJ79CoW9XqahRngIzEw_BDSq5HlQ-GPc3gYSzYs,1620
+zmq/tests/__init__.py,sha256=U0V-_B2wSbo4XGxJTQ6EFhInOstNXaHsUcF7Fj9BXok,7940
+zmq/tests/__pycache__/__init__.cpython-310.pyc,,
+zmq/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+zmq/utils/__pycache__/__init__.cpython-310.pyc,,
+zmq/utils/__pycache__/garbage.cpython-310.pyc,,
+zmq/utils/__pycache__/interop.cpython-310.pyc,,
+zmq/utils/__pycache__/jsonapi.cpython-310.pyc,,
+zmq/utils/__pycache__/monitor.cpython-310.pyc,,
+zmq/utils/__pycache__/strtypes.cpython-310.pyc,,
+zmq/utils/__pycache__/win32.cpython-310.pyc,,
+zmq/utils/__pycache__/z85.cpython-310.pyc,,
+zmq/utils/garbage.py,sha256=hfBcYNhJum7TcW4ktHJWa7CoNnDvSRFc4K5IK7IguWY,6124
+zmq/utils/getpid_compat.h,sha256=emvckPfSlYeCoUNgfYTkAWC4ie-LXLRnXDNLlXxXaPI,116
+zmq/utils/interop.py,sha256=l4AsLmDz3UHmuHjwc5EEZ61P_56HGVIVg88Lj4wnjPk,685
+zmq/utils/ipcmaxlen.h,sha256=q-YGX5BECL_QpOzOx3oC_I8mcNCWbJJ6FnUrdKlG1fU,522
+zmq/utils/jsonapi.py,sha256=2G1kMc3EW_Y4jSH_DwZzSti8lxzpRTx02kLrDokVDSA,1025
+zmq/utils/monitor.py,sha256=YVSdus-G9fwDmrGt2KtFj6GJIdzidq3Yvp6FZOyQFQ4,3287
+zmq/utils/mutex.h,sha256=SU-nEHeFNc_mzxQcDb9c8NpdLJTLpyECpy1GyYFpOFE,1646
+zmq/utils/pyversion_compat.h,sha256=4FkQ95UVmA_as9lBrIO7-wM5D0tEinVAlYmZls_SRT0,284
+zmq/utils/strtypes.py,sha256=sd0-cJGuDntYAcBMO_uqWAwvsOJGp8WXudMz3nJRUUA,1376
+zmq/utils/win32.py,sha256=aBQFhfZuNXvkHEtiYx5cqCAl3Mkl0qjsRzloIwR2K90,4940
+zmq/utils/z85.py,sha256=AM_l4fxgzA823RETaKj1QR8ci3gto-FoqvRm4qzxDXI,1838
+zmq/utils/zmq_compat.h,sha256=tXnvPnhIJZWG6pcKjbsj_2c7RpSwRFp9LDsXA8MQpSQ,2761
diff --git a/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/WHEEL b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/WHEEL
new file mode 100644
index 0000000000000000000000000000000000000000..fa0570a1cd512b337a072220fa072bbdcde4aaf8
--- /dev/null
+++ b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/WHEEL
@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: scikit-build-core 0.11.5
+Root-Is-Purelib: false
+Tag: cp310-cp310-manylinux_2_26_x86_64
+Tag: cp310-cp310-manylinux_2_28_x86_64
+
diff --git a/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/licenses/LICENSE.md b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/licenses/LICENSE.md
new file mode 100644
index 0000000000000000000000000000000000000000..f7072d1c994fbb6eb8a48365be0d583cb02a03e9
--- /dev/null
+++ b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/licenses/LICENSE.md
@@ -0,0 +1,30 @@
+BSD 3-Clause License
+
+Copyright (c) 2009-2012, Brian Granger, Min Ragan-Kelley
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/licenses/licenses/LICENSE.libsodium.txt b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/licenses/licenses/LICENSE.libsodium.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6bce7f0581c80aa6d05d33dd5267e1e047c738a9
--- /dev/null
+++ b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/licenses/licenses/LICENSE.libsodium.txt
@@ -0,0 +1,18 @@
+/*
+ * ISC License
+ *
+ * Copyright (c) 2013-2024
+ * Frank Denis 
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
diff --git a/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/licenses/licenses/LICENSE.tornado.txt b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/licenses/licenses/LICENSE.tornado.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7
--- /dev/null
+++ b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/licenses/licenses/LICENSE.tornado.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/licenses/licenses/LICENSE.zeromq.txt b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/licenses/licenses/LICENSE.zeromq.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a612ad9813b006ce81d1ee438dd784da99a54007
--- /dev/null
+++ b/lib/python3.1/site-packages/pyzmq-27.0.1.dist-info/licenses/licenses/LICENSE.zeromq.txt
@@ -0,0 +1,373 @@
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.
diff --git a/lib/python3.1/site-packages/ruff-0.12.9.dist-info/INSTALLER b/lib/python3.1/site-packages/ruff-0.12.9.dist-info/INSTALLER
new file mode 100644
index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68
--- /dev/null
+++ b/lib/python3.1/site-packages/ruff-0.12.9.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/lib/python3.1/site-packages/ruff-0.12.9.dist-info/METADATA b/lib/python3.1/site-packages/ruff-0.12.9.dist-info/METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..854d08bca85dec01f41a578fae998f86af8694a4
--- /dev/null
+++ b/lib/python3.1/site-packages/ruff-0.12.9.dist-info/METADATA
@@ -0,0 +1,580 @@
+Metadata-Version: 2.4
+Name: ruff
+Version: 0.12.9
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Rust
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Quality Assurance
+License-File: LICENSE
+Summary: An extremely fast Python linter and code formatter, written in Rust.
+Keywords: automation,flake8,pycodestyle,pyflakes,pylint,clippy
+Home-Page: https://docs.astral.sh/ruff
+Author-email: "Astral Software Inc." 
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Repository, https://github.com/astral-sh/ruff
+Project-URL: Documentation, https://docs.astral.sh/ruff/
+Project-URL: Changelog, https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md
+
+
+
+# Ruff
+
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![image](https://img.shields.io/pypi/v/ruff.svg)](https://pypi.python.org/pypi/ruff)
+[![image](https://img.shields.io/pypi/l/ruff.svg)](https://github.com/astral-sh/ruff/blob/main/LICENSE)
+[![image](https://img.shields.io/pypi/pyversions/ruff.svg)](https://pypi.python.org/pypi/ruff)
+[![Actions status](https://github.com/astral-sh/ruff/workflows/CI/badge.svg)](https://github.com/astral-sh/ruff/actions)
+[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?logo=discord&logoColor=white)](https://discord.com/invite/astral-sh)
+
+[**Docs**](https://docs.astral.sh/ruff/) | [**Playground**](https://play.ruff.rs/)
+
+An extremely fast Python linter and code formatter, written in Rust.
+
+
    
+  Shows a bar chart with benchmark results.
+
    
+
+
    
+  Linting the CPython codebase from scratch.
+
    
+
+- ⚡️ 10-100x faster than existing linters (like Flake8) and formatters (like Black)
+- 🐍 Installable via `pip`
+- 🛠️ `pyproject.toml` support
+- 🤝 Python 3.13 compatibility
+- ⚖️ Drop-in parity with [Flake8](https://docs.astral.sh/ruff/faq/#how-does-ruffs-linter-compare-to-flake8), isort, and [Black](https://docs.astral.sh/ruff/faq/#how-does-ruffs-formatter-compare-to-black)
+- 📦 Built-in caching, to avoid re-analyzing unchanged files
+- 🔧 Fix support, for automatic error correction (e.g., automatically remove unused imports)
+- 📏 Over [800 built-in rules](https://docs.astral.sh/ruff/rules/), with native re-implementations
+    of popular Flake8 plugins, like flake8-bugbear
+- ⌨️ First-party [editor integrations](https://docs.astral.sh/ruff/editors) for [VS Code](https://github.com/astral-sh/ruff-vscode) and [more](https://docs.astral.sh/ruff/editors/setup)
+- 🌎 Monorepo-friendly, with [hierarchical and cascading configuration](https://docs.astral.sh/ruff/configuration/#config-file-discovery)
+
+Ruff aims to be orders of magnitude faster than alternative tools while integrating more
+functionality behind a single, common interface.
+
+Ruff can be used to replace [Flake8](https://pypi.org/project/flake8/) (plus dozens of plugins),
+[Black](https://github.com/psf/black), [isort](https://pypi.org/project/isort/),
+[pydocstyle](https://pypi.org/project/pydocstyle/), [pyupgrade](https://pypi.org/project/pyupgrade/),
+[autoflake](https://pypi.org/project/autoflake/), and more, all while executing tens or hundreds of
+times faster than any individual tool.
+
+Ruff is extremely actively developed and used in major open-source projects like:
+
+- [Apache Airflow](https://github.com/apache/airflow)
+- [Apache Superset](https://github.com/apache/superset)
+- [FastAPI](https://github.com/tiangolo/fastapi)
+- [Hugging Face](https://github.com/huggingface/transformers)
+- [Pandas](https://github.com/pandas-dev/pandas)
+- [SciPy](https://github.com/scipy/scipy)
+
+...and [many more](#whos-using-ruff).
+
+Ruff is backed by [Astral](https://astral.sh). Read the [launch post](https://astral.sh/blog/announcing-astral-the-company-behind-ruff),
+or the original [project announcement](https://notes.crmarsh.com/python-tooling-could-be-much-much-faster).
+
+## Testimonials
+
+[**Sebastián Ramírez**](https://twitter.com/tiangolo/status/1591912354882764802), creator
+of [FastAPI](https://github.com/tiangolo/fastapi):
+
+> Ruff is so fast that sometimes I add an intentional bug in the code just to confirm it's actually
+> running and checking the code.
+
+[**Nick Schrock**](https://twitter.com/schrockn/status/1612615862904827904), founder of [Elementl](https://www.elementl.com/),
+co-creator of [GraphQL](https://graphql.org/):
+
+> Why is Ruff a gamechanger? Primarily because it is nearly 1000x faster. Literally. Not a typo. On
+> our largest module (dagster itself, 250k LOC) pylint takes about 2.5 minutes, parallelized across 4
+> cores on my M1. Running ruff against our _entire_ codebase takes .4 seconds.
+
+[**Bryan Van de Ven**](https://github.com/bokeh/bokeh/pull/12605), co-creator
+of [Bokeh](https://github.com/bokeh/bokeh/), original author
+of [Conda](https://docs.conda.io/en/latest/):
+
+> Ruff is ~150-200x faster than flake8 on my machine, scanning the whole repo takes ~0.2s instead of
+> ~20s. This is an enormous quality of life improvement for local dev. It's fast enough that I added
+> it as an actual commit hook, which is terrific.
+
+[**Timothy Crosley**](https://twitter.com/timothycrosley/status/1606420868514877440),
+creator of [isort](https://github.com/PyCQA/isort):
+
+> Just switched my first project to Ruff. Only one downside so far: it's so fast I couldn't believe
+> it was working till I intentionally introduced some errors.
+
+[**Tim Abbott**](https://github.com/astral-sh/ruff/issues/465#issuecomment-1317400028), lead
+developer of [Zulip](https://github.com/zulip/zulip):
+
+> This is just ridiculously fast... `ruff` is amazing.
+
+
+
+## Table of Contents
+
+For more, see the [documentation](https://docs.astral.sh/ruff/).
+
+1. [Getting Started](#getting-started)
+1. [Configuration](#configuration)
+1. [Rules](#rules)
+1. [Contributing](#contributing)
+1. [Support](#support)
+1. [Acknowledgements](#acknowledgements)
+1. [Who's Using Ruff?](#whos-using-ruff)
+1. [License](#license)
+
+## Getting Started
+
+For more, see the [documentation](https://docs.astral.sh/ruff/).
+
+### Installation
+
+Ruff is available as [`ruff`](https://pypi.org/project/ruff/) on PyPI.
+
+Invoke Ruff directly with [`uvx`](https://docs.astral.sh/uv/):
+
+```shell
+uvx ruff check   # Lint all files in the current directory.
+uvx ruff format  # Format all files in the current directory.
+```
+
+Or install Ruff with `uv` (recommended), `pip`, or `pipx`:
+
+```shell
+# With uv.
+uv tool install ruff@latest  # Install Ruff globally.
+uv add --dev ruff            # Or add Ruff to your project.
+
+# With pip.
+pip install ruff
+
+# With pipx.
+pipx install ruff
+```
+
+Starting with version `0.5.0`, Ruff can be installed with our standalone installers:
+
+```shell
+# On macOS and Linux.
+curl -LsSf https://astral.sh/ruff/install.sh | sh
+
+# On Windows.
+powershell -c "irm https://astral.sh/ruff/install.ps1 | iex"
+
+# For a specific version.
+curl -LsSf https://astral.sh/ruff/0.12.9/install.sh | sh
+powershell -c "irm https://astral.sh/ruff/0.12.9/install.ps1 | iex"
+```
+
+You can also install Ruff via [Homebrew](https://formulae.brew.sh/formula/ruff), [Conda](https://anaconda.org/conda-forge/ruff),
+and with [a variety of other package managers](https://docs.astral.sh/ruff/installation/).
+
+### Usage
+
+To run Ruff as a linter, try any of the following:
+
+```shell
+ruff check                          # Lint all files in the current directory (and any subdirectories).
+ruff check path/to/code/            # Lint all files in `/path/to/code` (and any subdirectories).
+ruff check path/to/code/*.py        # Lint all `.py` files in `/path/to/code`.
+ruff check path/to/code/to/file.py  # Lint `file.py`.
+ruff check @arguments.txt           # Lint using an input file, treating its contents as newline-delimited command-line arguments.
+```
+
+Or, to run Ruff as a formatter:
+
+```shell
+ruff format                          # Format all files in the current directory (and any subdirectories).
+ruff format path/to/code/            # Format all files in `/path/to/code` (and any subdirectories).
+ruff format path/to/code/*.py        # Format all `.py` files in `/path/to/code`.
+ruff format path/to/code/to/file.py  # Format `file.py`.
+ruff format @arguments.txt           # Format using an input file, treating its contents as newline-delimited command-line arguments.
+```
+
+Ruff can also be used as a [pre-commit](https://pre-commit.com/) hook via [`ruff-pre-commit`](https://github.com/astral-sh/ruff-pre-commit):
+
+```yaml
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.12.9
+  hooks:
+    # Run the linter.
+    - id: ruff-check
+      args: [ --fix ]
+    # Run the formatter.
+    - id: ruff-format
+```
+
+Ruff can also be used as a [VS Code extension](https://github.com/astral-sh/ruff-vscode) or with [various other editors](https://docs.astral.sh/ruff/editors/setup).
+
+Ruff can also be used as a [GitHub Action](https://github.com/features/actions) via
+[`ruff-action`](https://github.com/astral-sh/ruff-action):
+
+```yaml
+name: Ruff
+on: [ push, pull_request ]
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/ruff-action@v3
+```
+
+### Configuration
+
+Ruff can be configured through a `pyproject.toml`, `ruff.toml`, or `.ruff.toml` file (see:
+[_Configuration_](https://docs.astral.sh/ruff/configuration/), or [_Settings_](https://docs.astral.sh/ruff/settings/)
+for a complete list of all configuration options).
+
+If left unspecified, Ruff's default configuration is equivalent to the following `ruff.toml` file:
+
+```toml
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pyenv",
+    ".pytest_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+]
+
+# Same as Black.
+line-length = 88
+indent-width = 4
+
+# Assume Python 3.9
+target-version = "py39"
+
+[lint]
+# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default.
+select = ["E4", "E7", "E9", "F"]
+ignore = []
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[format]
+# Like Black, use double quotes for strings.
+quote-style = "double"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"
+```
+
+Note that, in a `pyproject.toml`, each section header should be prefixed with `tool.ruff`. For
+example, `[lint]` should be replaced with `[tool.ruff.lint]`.
+
+Some configuration options can be provided via dedicated command-line arguments, such as those
+related to rule enablement and disablement, file discovery, and logging level:
+
+```shell
+ruff check --select F401 --select F403 --quiet
+```
+
+The remaining configuration options can be provided through a catch-all `--config` argument:
+
+```shell
+ruff check --config "lint.per-file-ignores = {'some_file.py' = ['F841']}"
+```
+
+To opt in to the latest lint rules, formatter style changes, interface updates, and more, enable
+[preview mode](https://docs.astral.sh/ruff/rules/) by setting `preview = true` in your configuration
+file or passing `--preview` on the command line. Preview mode enables a collection of unstable
+features that may change prior to stabilization.
+
+See `ruff help` for more on Ruff's top-level commands, or `ruff help check` and `ruff help format`
+for more on the linting and formatting commands, respectively.
+
+## Rules
+
+
+
+**Ruff supports over 800 lint rules**, many of which are inspired by popular tools like Flake8,
+isort, pyupgrade, and others. Regardless of the rule's origin, Ruff re-implements every rule in
+Rust as a first-party feature.
+
+By default, Ruff enables Flake8's `F` rules, along with a subset of the `E` rules, omitting any
+stylistic rules that overlap with the use of a formatter, like `ruff format` or
+[Black](https://github.com/psf/black).
+
+If you're just getting started with Ruff, **the default rule set is a great place to start**: it
+catches a wide variety of common errors (like unused imports) with zero configuration.
+
+
+
+Beyond the defaults, Ruff re-implements some of the most popular Flake8 plugins and related code
+quality tools, including:
+
+- [autoflake](https://pypi.org/project/autoflake/)
+- [eradicate](https://pypi.org/project/eradicate/)
+- [flake8-2020](https://pypi.org/project/flake8-2020/)
+- [flake8-annotations](https://pypi.org/project/flake8-annotations/)
+- [flake8-async](https://pypi.org/project/flake8-async)
+- [flake8-bandit](https://pypi.org/project/flake8-bandit/) ([#1646](https://github.com/astral-sh/ruff/issues/1646))
+- [flake8-blind-except](https://pypi.org/project/flake8-blind-except/)
+- [flake8-boolean-trap](https://pypi.org/project/flake8-boolean-trap/)
+- [flake8-bugbear](https://pypi.org/project/flake8-bugbear/)
+- [flake8-builtins](https://pypi.org/project/flake8-builtins/)
+- [flake8-commas](https://pypi.org/project/flake8-commas/)
+- [flake8-comprehensions](https://pypi.org/project/flake8-comprehensions/)
+- [flake8-copyright](https://pypi.org/project/flake8-copyright/)
+- [flake8-datetimez](https://pypi.org/project/flake8-datetimez/)
+- [flake8-debugger](https://pypi.org/project/flake8-debugger/)
+- [flake8-django](https://pypi.org/project/flake8-django/)
+- [flake8-docstrings](https://pypi.org/project/flake8-docstrings/)
+- [flake8-eradicate](https://pypi.org/project/flake8-eradicate/)
+- [flake8-errmsg](https://pypi.org/project/flake8-errmsg/)
+- [flake8-executable](https://pypi.org/project/flake8-executable/)
+- [flake8-future-annotations](https://pypi.org/project/flake8-future-annotations/)
+- [flake8-gettext](https://pypi.org/project/flake8-gettext/)
+- [flake8-implicit-str-concat](https://pypi.org/project/flake8-implicit-str-concat/)
+- [flake8-import-conventions](https://github.com/joaopalmeiro/flake8-import-conventions)
+- [flake8-logging](https://pypi.org/project/flake8-logging/)
+- [flake8-logging-format](https://pypi.org/project/flake8-logging-format/)
+- [flake8-no-pep420](https://pypi.org/project/flake8-no-pep420)
+- [flake8-pie](https://pypi.org/project/flake8-pie/)
+- [flake8-print](https://pypi.org/project/flake8-print/)
+- [flake8-pyi](https://pypi.org/project/flake8-pyi/)
+- [flake8-pytest-style](https://pypi.org/project/flake8-pytest-style/)
+- [flake8-quotes](https://pypi.org/project/flake8-quotes/)
+- [flake8-raise](https://pypi.org/project/flake8-raise/)
+- [flake8-return](https://pypi.org/project/flake8-return/)
+- [flake8-self](https://pypi.org/project/flake8-self/)
+- [flake8-simplify](https://pypi.org/project/flake8-simplify/)
+- [flake8-slots](https://pypi.org/project/flake8-slots/)
+- [flake8-super](https://pypi.org/project/flake8-super/)
+- [flake8-tidy-imports](https://pypi.org/project/flake8-tidy-imports/)
+- [flake8-todos](https://pypi.org/project/flake8-todos/)
+- [flake8-type-checking](https://pypi.org/project/flake8-type-checking/)
+- [flake8-use-pathlib](https://pypi.org/project/flake8-use-pathlib/)
+- [flynt](https://pypi.org/project/flynt/) ([#2102](https://github.com/astral-sh/ruff/issues/2102))
+- [isort](https://pypi.org/project/isort/)
+- [mccabe](https://pypi.org/project/mccabe/)
+- [pandas-vet](https://pypi.org/project/pandas-vet/)
+- [pep8-naming](https://pypi.org/project/pep8-naming/)
+- [pydocstyle](https://pypi.org/project/pydocstyle/)
+- [pygrep-hooks](https://github.com/pre-commit/pygrep-hooks)
+- [pylint-airflow](https://pypi.org/project/pylint-airflow/)
+- [pyupgrade](https://pypi.org/project/pyupgrade/)
+- [tryceratops](https://pypi.org/project/tryceratops/)
+- [yesqa](https://pypi.org/project/yesqa/)
+
+For a complete enumeration of the supported rules, see [_Rules_](https://docs.astral.sh/ruff/rules/).
+
+## Contributing
+
+Contributions are welcome and highly appreciated. To get started, check out the
+[**contributing guidelines**](https://docs.astral.sh/ruff/contributing/).
+
+You can also join us on [**Discord**](https://discord.com/invite/astral-sh).
+
+## Support
+
+Having trouble? Check out the existing issues on [**GitHub**](https://github.com/astral-sh/ruff/issues),
+or feel free to [**open a new one**](https://github.com/astral-sh/ruff/issues/new).
+
+You can also ask for help on [**Discord**](https://discord.com/invite/astral-sh).
+
+## Acknowledgements
+
+Ruff's linter draws on both the APIs and implementation details of many other
+tools in the Python ecosystem, especially [Flake8](https://github.com/PyCQA/flake8), [Pyflakes](https://github.com/PyCQA/pyflakes),
+[pycodestyle](https://github.com/PyCQA/pycodestyle), [pydocstyle](https://github.com/PyCQA/pydocstyle),
+[pyupgrade](https://github.com/asottile/pyupgrade), and [isort](https://github.com/PyCQA/isort).
+
+In some cases, Ruff includes a "direct" Rust port of the corresponding tool.
+We're grateful to the maintainers of these tools for their work, and for all
+the value they've provided to the Python community.
+
+Ruff's formatter is built on a fork of Rome's [`rome_formatter`](https://github.com/rome/tools/tree/main/crates/rome_formatter),
+and again draws on both API and implementation details from [Rome](https://github.com/rome/tools),
+[Prettier](https://github.com/prettier/prettier), and [Black](https://github.com/psf/black).
+
+Ruff's import resolver is based on the import resolution algorithm from [Pyright](https://github.com/microsoft/pyright).
+
+Ruff is also influenced by a number of tools outside the Python ecosystem, like
+[Clippy](https://github.com/rust-lang/rust-clippy) and [ESLint](https://github.com/eslint/eslint).
+
+Ruff is the beneficiary of a large number of [contributors](https://github.com/astral-sh/ruff/graphs/contributors).
+
+Ruff is released under the MIT license.
+
+## Who's Using Ruff?
+
+Ruff is used by a number of major open-source projects and companies, including:
+
+- [Albumentations](https://github.com/albumentations-team/albumentations)
+- Amazon ([AWS SAM](https://github.com/aws/serverless-application-model))
+- [Anki](https://apps.ankiweb.net/)
+- Anthropic ([Python SDK](https://github.com/anthropics/anthropic-sdk-python))
+- [Apache Airflow](https://github.com/apache/airflow)
+- AstraZeneca ([Magnus](https://github.com/AstraZeneca/magnus-core))
+- [Babel](https://github.com/python-babel/babel)
+- Benchling ([Refac](https://github.com/benchling/refac))
+- [Bokeh](https://github.com/bokeh/bokeh)
+- Capital One ([datacompy](https://github.com/capitalone/datacompy))
+- CrowdCent ([NumerBlox](https://github.com/crowdcent/numerblox)) 
+- [Cryptography (PyCA)](https://github.com/pyca/cryptography)
+- CERN ([Indico](https://getindico.io/))
+- [DVC](https://github.com/iterative/dvc)
+- [Dagger](https://github.com/dagger/dagger)
+- [Dagster](https://github.com/dagster-io/dagster)
+- Databricks ([MLflow](https://github.com/mlflow/mlflow))
+- [Dify](https://github.com/langgenius/dify)
+- [FastAPI](https://github.com/tiangolo/fastapi)
+- [Godot](https://github.com/godotengine/godot)
+- [Gradio](https://github.com/gradio-app/gradio)
+- [Great Expectations](https://github.com/great-expectations/great_expectations)
+- [HTTPX](https://github.com/encode/httpx)
+- [Hatch](https://github.com/pypa/hatch)
+- [Home Assistant](https://github.com/home-assistant/core)
+- Hugging Face ([Transformers](https://github.com/huggingface/transformers),
+    [Datasets](https://github.com/huggingface/datasets),
+    [Diffusers](https://github.com/huggingface/diffusers))
+- IBM ([Qiskit](https://github.com/Qiskit/qiskit))
+- ING Bank ([popmon](https://github.com/ing-bank/popmon), [probatus](https://github.com/ing-bank/probatus))
+- [Ibis](https://github.com/ibis-project/ibis)
+- [ivy](https://github.com/unifyai/ivy)
+- [JAX](https://github.com/jax-ml/jax)
+- [Jupyter](https://github.com/jupyter-server/jupyter_server)
+- [Kraken Tech](https://kraken.tech/)
+- [LangChain](https://github.com/hwchase17/langchain)
+- [Litestar](https://litestar.dev/)
+- [LlamaIndex](https://github.com/jerryjliu/llama_index)
+- Matrix ([Synapse](https://github.com/matrix-org/synapse))
+- [MegaLinter](https://github.com/oxsecurity/megalinter)
+- Meltano ([Meltano CLI](https://github.com/meltano/meltano), [Singer SDK](https://github.com/meltano/sdk))
+- Microsoft ([Semantic Kernel](https://github.com/microsoft/semantic-kernel),
+    [ONNX Runtime](https://github.com/microsoft/onnxruntime),
+    [LightGBM](https://github.com/microsoft/LightGBM))
+- Modern Treasury ([Python SDK](https://github.com/Modern-Treasury/modern-treasury-python))
+- Mozilla ([Firefox](https://github.com/mozilla/gecko-dev))
+- [Mypy](https://github.com/python/mypy)
+- [Nautobot](https://github.com/nautobot/nautobot)
+- Netflix ([Dispatch](https://github.com/Netflix/dispatch))
+- [Neon](https://github.com/neondatabase/neon)
+- [Nokia](https://nokia.com/)
+- [NoneBot](https://github.com/nonebot/nonebot2)
+- [NumPyro](https://github.com/pyro-ppl/numpyro)
+- [ONNX](https://github.com/onnx/onnx)
+- [OpenBB](https://github.com/OpenBB-finance/OpenBBTerminal)
+- [Open Wine Components](https://github.com/Open-Wine-Components/umu-launcher)
+- [PDM](https://github.com/pdm-project/pdm)
+- [PaddlePaddle](https://github.com/PaddlePaddle/Paddle)
+- [Pandas](https://github.com/pandas-dev/pandas)
+- [Pillow](https://github.com/python-pillow/Pillow)
+- [Poetry](https://github.com/python-poetry/poetry)
+- [Polars](https://github.com/pola-rs/polars)
+- [PostHog](https://github.com/PostHog/posthog)
+- Prefect ([Python SDK](https://github.com/PrefectHQ/prefect), [Marvin](https://github.com/PrefectHQ/marvin))
+- [PyInstaller](https://github.com/pyinstaller/pyinstaller)
+- [PyMC](https://github.com/pymc-devs/pymc/)
+- [PyMC-Marketing](https://github.com/pymc-labs/pymc-marketing)
+- [pytest](https://github.com/pytest-dev/pytest)
+- [PyTorch](https://github.com/pytorch/pytorch)
+- [Pydantic](https://github.com/pydantic/pydantic)
+- [Pylint](https://github.com/PyCQA/pylint)
+- [PyVista](https://github.com/pyvista/pyvista)
+- [Reflex](https://github.com/reflex-dev/reflex)
+- [River](https://github.com/online-ml/river)
+- [Rippling](https://rippling.com)
+- [Robyn](https://github.com/sansyrox/robyn)
+- [Saleor](https://github.com/saleor/saleor)
+- Scale AI ([Launch SDK](https://github.com/scaleapi/launch-python-client))
+- [SciPy](https://github.com/scipy/scipy)
+- Snowflake ([SnowCLI](https://github.com/Snowflake-Labs/snowcli))
+- [Sphinx](https://github.com/sphinx-doc/sphinx)
+- [Stable Baselines3](https://github.com/DLR-RM/stable-baselines3)
+- [Starlette](https://github.com/encode/starlette)
+- [Streamlit](https://github.com/streamlit/streamlit)
+- [The Algorithms](https://github.com/TheAlgorithms/Python)
+- [Vega-Altair](https://github.com/altair-viz/altair)
+- [Weblate](https://weblate.org/)
+- WordPress ([Openverse](https://github.com/WordPress/openverse))
+- [ZenML](https://github.com/zenml-io/zenml)
+- [Zulip](https://github.com/zulip/zulip)
+- [build (PyPA)](https://github.com/pypa/build)
+- [cibuildwheel (PyPA)](https://github.com/pypa/cibuildwheel)
+- [delta-rs](https://github.com/delta-io/delta-rs)
+- [featuretools](https://github.com/alteryx/featuretools)
+- [meson-python](https://github.com/mesonbuild/meson-python)
+- [nox](https://github.com/wntrblm/nox)
+- [pip](https://github.com/pypa/pip)
+
+### Show Your Support
+
+If you're using Ruff, consider adding the Ruff badge to your project's `README.md`:
+
+```md
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+```
+
+...or `README.rst`:
+
+```rst
+.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json
+    :target: https://github.com/astral-sh/ruff
+    :alt: Ruff
+```
+
+...or, as HTML:
+
+```html
+Ruff
+```
+
+## License
+
+This repository is licensed under the [MIT License](https://github.com/astral-sh/ruff/blob/main/LICENSE)
+
+
+
diff --git a/lib/python3.1/site-packages/ruff-0.12.9.dist-info/RECORD b/lib/python3.1/site-packages/ruff-0.12.9.dist-info/RECORD
new file mode 100644
index 0000000000000000000000000000000000000000..d563d6679e2fc847178c12bfdd5aaf49202ef07a
--- /dev/null
+++ b/lib/python3.1/site-packages/ruff-0.12.9.dist-info/RECORD
@@ -0,0 +1,10 @@
+../../../bin/ruff,sha256=znbdEx64LZgsXUvB0E8_4prGJAiKpEpAVj1Ek8mrBE0,35480752
+ruff-0.12.9.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+ruff-0.12.9.dist-info/METADATA,sha256=BJUMuB1R6Jsr9qRP2WnJIQpE2urzOtMTIUrTZi8cuM8,25808
+ruff-0.12.9.dist-info/RECORD,,
+ruff-0.12.9.dist-info/WHEEL,sha256=l495f3Ss-y5Wd7Mtx9LlysmWOrt2fkUq2-Qpvz3qRdI,126
+ruff-0.12.9.dist-info/licenses/LICENSE,sha256=MgnAt_tyV8BbFkM3KMHpdLrEAAD-8XG46eifCVYjyVQ,69789
+ruff/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ruff/__main__.py,sha256=w1Ehfzzy8tMM7r7dFrKmi3DkmTZyV_xJ40K_sxoO6X0,2908
+ruff/__pycache__/__init__.cpython-310.pyc,,
+ruff/__pycache__/__main__.cpython-310.pyc,,
diff --git a/lib/python3.1/site-packages/ruff-0.12.9.dist-info/WHEEL b/lib/python3.1/site-packages/ruff-0.12.9.dist-info/WHEEL
new file mode 100644
index 0000000000000000000000000000000000000000..7329f78830437a5806f78ce5344aa92763244860
--- /dev/null
+++ b/lib/python3.1/site-packages/ruff-0.12.9.dist-info/WHEEL
@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.9.3)
+Root-Is-Purelib: false
+Tag: py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64
diff --git a/lib/python3.1/site-packages/ruff-0.12.9.dist-info/licenses/LICENSE b/lib/python3.1/site-packages/ruff-0.12.9.dist-info/licenses/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..f5c3b02beccb0b271fe0873359715722ad15493c
--- /dev/null
+++ b/lib/python3.1/site-packages/ruff-0.12.9.dist-info/licenses/LICENSE
@@ -0,0 +1,1398 @@
+MIT License
+
+Copyright (c) 2022 Charles Marsh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+end of terms and conditions
+
+The externally maintained libraries from which parts of the Software is derived
+are:
+
+- flake8-comprehensions, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2017 Adam Johnson
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-no-pep420, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2020 Adam Johnson
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-tidy-imports, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2017 Adam Johnson
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-return, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2019 Afonasev Evgeniy
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-2020, licensed as follows:
+  """
+    Copyright (c) 2019 Anthony Sottile
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+  """
+
+- pyupgrade, licensed as follows:
+  """
+    Copyright (c) 2017 Anthony Sottile
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+  """
+
+- flake8-blind-except, licensed as follows:
+  """
+    The MIT License (MIT)
+
+    Copyright (c) 2014 Elijah Andrews
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy of
+    this software and associated documentation files (the "Software"), to deal in
+    the Software without restriction, including without limitation the rights to
+    use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+    the Software, and to permit persons to whom the Software is furnished to do so,
+    subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+    FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+    COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+    IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+    CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+  """
+
+- flake8-gettext, licensed as follows:
+  """
+    BSD Zero Clause License
+
+    Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted.
+
+    THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+  """
+
+- flake8-implicit-str-concat, licensed as follows:
+  """
+    The MIT License (MIT)
+
+    Copyright (c) 2019 Dylan Turner
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+  """
+
+- flake8-debugger, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2016 Joseph Kahn
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-pyi, licensed as follows:
+  """
+    The MIT License (MIT)
+
+    Copyright (c) 2016 Łukasz Langa
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-print, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2016 Joseph Kahn
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-import-conventions, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2021 João Palmeiro
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-simplify, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2020 Martin Thoma
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-slots, licensed as follows:
+  """
+    Copyright (c) 2021 Dominic Davis-Foster
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+    IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+    DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+    OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
+    OR OTHER DEALINGS IN THE SOFTWARE.
+  """
+
+- flake8-todos, licensed as follows:
+  """
+    Copyright (c) 2019 EclecticIQ. All rights reserved.
+    Copyright (c) 2020 Gram . All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+       1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+       2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+
+       3. Neither the name of the copyright holder nor the names of its
+    contributors may be used to endorse or promote products derived from this
+    software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+    AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+    IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+    DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+    FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+    OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+  """
+
+- flake8-unused-arguments, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2019 Nathan Hoad
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- pygrep-hooks, licensed as follows:
+  """
+    Copyright (c) 2018 Anthony Sottile
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+  """
+
+- autoflake, licensed as follows:
+  """
+    Copyright (C) 2012-2018 Steven Myint
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy of
+    this software and associated documentation files (the "Software"), to deal in
+    the Software without restriction, including without limitation the rights to
+    use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is furnished to do
+    so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- autotyping, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2023 Jelle Zijlstra
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- Flake8, licensed as follows:
+  """
+    == Flake8 License (MIT) ==
+
+    Copyright (C) 2011-2013 Tarek Ziade 
+    Copyright (C) 2012-2016 Ian Cordasco 
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy of
+    this software and associated documentation files (the "Software"), to deal in
+    the Software without restriction, including without limitation the rights to
+    use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is furnished to do
+    so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-bugbear, licensed as follows:
+  """
+    The MIT License (MIT)
+
+    Copyright (c) 2016 Łukasz Langa
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-commas, licensed as follows:
+  """
+    The MIT License (MIT)
+
+    Copyright (c) 2017 Thomas Grainger.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+
+
+    Portions of this flake8-commas Software may utilize the following
+    copyrighted material, the use of which is hereby acknowledged.
+
+    Original flake8-commas: https://github.com/trevorcreech/flake8-commas/commit/e8563b71b1d5442e102c8734c11cb5202284293d
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+  """
+
+- flynt, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2019-2022 Ilya Kamenshchikov
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- isort, licensed as follows:
+  """
+    The MIT License (MIT)
+
+    Copyright (c) 2013 Timothy Edmund Crosley
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+  """
+
+- pep8-naming, licensed as follows:
+  """
+    Copyright © 2013 Florent Xicluna 
+
+    Licensed under the terms of the Expat License
+
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation files
+    (the "Software"), to deal in the Software without restriction,
+    including without limitation the rights to use, copy, modify, merge,
+    publish, distribute, sublicense, and/or sell copies of the Software,
+    and to permit persons to whom the Software is furnished to do so,
+    subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+    BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+    ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+    CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- pycodestyle, licensed as follows:
+  """
+    Copyright © 2006-2009 Johann C. Rocholl 
+    Copyright © 2009-2014 Florent Xicluna 
+    Copyright © 2014-2020 Ian Lee 
+
+    Licensed under the terms of the Expat License
+
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation files
+    (the "Software"), to deal in the Software without restriction,
+    including without limitation the rights to use, copy, modify, merge,
+    publish, distribute, sublicense, and/or sell copies of the Software,
+    and to permit persons to whom the Software is furnished to do so,
+    subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+    BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+    ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+    CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- pydocstyle, licensed as follows:
+  """
+    Copyright (c) 2012 GreenSteam, 
+
+    Copyright (c) 2014-2020 Amir Rachum, 
+
+    Copyright (c) 2020 Sambhav Kothari, 
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy of
+    this software and associated documentation files (the "Software"), to deal in
+    the Software without restriction, including without limitation the rights to
+    use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+    of the Software, and to permit persons to whom the Software is furnished to do
+    so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- Pyflakes, licensed as follows:
+  """
+    Copyright 2005-2011 Divmod, Inc.
+    Copyright 2013-2014 Florent Xicluna
+
+    Permission is hereby granted, free of charge, to any person obtaining
+    a copy of this software and associated documentation files (the
+    "Software"), to deal in the Software without restriction, including
+    without limitation the rights to use, copy, modify, merge, publish,
+    distribute, sublicense, and/or sell copies of the Software, and to
+    permit persons to whom the Software is furnished to do so, subject to
+    the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+    LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+  """
+
+- flake8-use-pathlib, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2021 Rodolphe Pelloux-Prayer
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- RustPython, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2020 RustPython Team
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-annotations, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2019 - Present S. Co1
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-async, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2022 Cooper Lees
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-type-checking, licensed as follows:
+  """
+    Copyright (c) 2021, Sondre Lillebø Gundersen
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, this
+      list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+
+    * Neither the name of pytest-{{ cookiecutter.plugin_name }} nor the names of its
+      contributors may be used to endorse or promote products derived from
+      this software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+    AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+    IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+    DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+    FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+    OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+  """
+
+- flake8-bandit, licensed as follows:
+  """
+    Copyright (c) 2017 Tyler Wince
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+  """
+
+- flake8-eradicate, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2018 Nikita Sobolev
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-quotes, licensed as follows:
+  """
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+  """
+
+- flake8-logging-format, licensed as follows:
+  """
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+    1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+    2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+    3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+    4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+    5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+    6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+    7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+    8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+    9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+    END OF TERMS AND CONDITIONS
+
+    APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+    Copyright {yyyy} {name of copyright owner}
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+  """
+
+- flake8-raise, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2020 Jon Dufresne
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-self, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2023 Korijn van Golen
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-django, licensed under the GPL license.
+
+- perflint, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2022 Anthony Shaw
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-logging, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2023 Adam Johnson
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- flake8-trio, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2022 Zac Hatfield-Dodds
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- Pyright, licensed as follows:
+  """
+    MIT License
+
+    Pyright - A static type checker for the Python language
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE
+  """
+
+- rust-analyzer/text-size, licensed under the MIT license:
+  """
+    Permission is hereby granted, free of charge, to any
+    person obtaining a copy of this software and associated
+    documentation files (the "Software"), to deal in the
+    Software without restriction, including without
+    limitation the rights to use, copy, modify, merge,
+    publish, distribute, sublicense, and/or sell copies of
+    the Software, and to permit persons to whom the Software
+    is furnished to do so, subject to the following
+    conditions:
+
+    The above copyright notice and this permission notice
+    shall be included in all copies or substantial portions
+    of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+    ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+    TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+    PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+    SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+    CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+    IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+  """
+
+- rome/tools, licensed under the MIT license:
+  """
+    MIT License
+
+    Copyright (c) Rome Tools, Inc. and its affiliates.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
+
+- pydoclint, licensed as follows:
+  """
+    MIT License
+
+    Copyright (c) 2023 jsh9
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """
diff --git a/lib/python3.1/site-packages/ruff/__init__.py b/lib/python3.1/site-packages/ruff/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lib/python3.1/site-packages/ruff/__main__.py b/lib/python3.1/site-packages/ruff/__main__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8569844c4ed3452789110ec3e6efbcf307d3012d
--- /dev/null
+++ b/lib/python3.1/site-packages/ruff/__main__.py
@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import os
+import sys
+import sysconfig
+
+
+def find_ruff_bin() -> str:
+    """Return the ruff binary path."""
+
+    ruff_exe = "ruff" + sysconfig.get_config_var("EXE")
+
+    scripts_path = os.path.join(sysconfig.get_path("scripts"), ruff_exe)
+    if os.path.isfile(scripts_path):
+        return scripts_path
+
+    if sys.version_info >= (3, 10):
+        user_scheme = sysconfig.get_preferred_scheme("user")
+    elif os.name == "nt":
+        user_scheme = "nt_user"
+    elif sys.platform == "darwin" and sys._framework:
+        user_scheme = "osx_framework_user"
+    else:
+        user_scheme = "posix_user"
+
+    user_path = os.path.join(
+        sysconfig.get_path("scripts", scheme=user_scheme), ruff_exe
+    )
+    if os.path.isfile(user_path):
+        return user_path
+
+    # Search in `bin` adjacent to package root (as created by `pip install --target`).
+    pkg_root = os.path.dirname(os.path.dirname(__file__))
+    target_path = os.path.join(pkg_root, "bin", ruff_exe)
+    if os.path.isfile(target_path):
+        return target_path
+
+    # Search for pip-specific build environments.
+    #
+    # Expect to find ruff in /pip-build-env-/overlay/bin/ruff
+    # Expect to find a "normal" folder at /pip-build-env-/normal
+    #
+    # See: https://github.com/pypa/pip/blob/102d8187a1f5a4cd5de7a549fd8a9af34e89a54f/src/pip/_internal/build_env.py#L87
+    paths = os.environ.get("PATH", "").split(os.pathsep)
+    if len(paths) >= 2:
+
+        def get_last_three_path_parts(path: str) -> list[str]:
+            """Return a list of up to the last three parts of a path."""
+            parts = []
+
+            while len(parts) < 3:
+                head, tail = os.path.split(path)
+                if tail or head != path:
+                    parts.append(tail)
+                    path = head
+                else:
+                    parts.append(path)
+                    break
+
+            return parts
+
+        maybe_overlay = get_last_three_path_parts(paths[0])
+        maybe_normal = get_last_three_path_parts(paths[1])
+        if (
+            len(maybe_normal) >= 3
+            and maybe_normal[-1].startswith("pip-build-env-")
+            and maybe_normal[-2] == "normal"
+            and len(maybe_overlay) >= 3
+            and maybe_overlay[-1].startswith("pip-build-env-")
+            and maybe_overlay[-2] == "overlay"
+        ):
+            # The overlay must contain the ruff binary.
+            candidate = os.path.join(paths[0], ruff_exe)
+            if os.path.isfile(candidate):
+                return candidate
+
+    raise FileNotFoundError(scripts_path)
+
+
+if __name__ == "__main__":
+    ruff = find_ruff_bin()
+    if sys.platform == "win32":
+        import subprocess
+
+        completed_process = subprocess.run([ruff, *sys.argv[1:]])
+        sys.exit(completed_process.returncode)
+    else:
+        os.execvp(ruff, [ruff, *sys.argv[1:]])
diff --git a/lib/python3.1/site-packages/ruff/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/site-packages/ruff/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f1ba24d61030ad8f25dd3e8101f28099b4ce49db
Binary files /dev/null and b/lib/python3.1/site-packages/ruff/__pycache__/__init__.cpython-310.pyc differ
diff --git a/lib/python3.1/site-packages/ruff/__pycache__/__main__.cpython-310.pyc b/lib/python3.1/site-packages/ruff/__pycache__/__main__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ee913ab9ffe9b0ee4f093e5a1ad749cf43a1f0d3
Binary files /dev/null and b/lib/python3.1/site-packages/ruff/__pycache__/__main__.cpython-310.pyc differ
diff --git a/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/INSTALLER b/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/INSTALLER
new file mode 100644
index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68
--- /dev/null
+++ b/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/LICENSE b/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..66aba18cab4ec4c67e7adbd4f722953231c1d33e
--- /dev/null
+++ b/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/LICENSE
@@ -0,0 +1,22 @@
+Copyright (c) The python-semanticversion project
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met: 
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer. 
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution. 
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/METADATA b/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..d9d9d8a31337735dea30657a80a58462dfcffd26
--- /dev/null
+++ b/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/METADATA
@@ -0,0 +1,324 @@
+Metadata-Version: 2.1
+Name: semantic-version
+Version: 2.10.0
+Summary: A library implementing the 'SemVer' scheme.
+Home-page: https://github.com/rbarrois/python-semanticversion
+Author: Raphaël Barrois
+Author-email: raphael.barrois+semver@polytechnique.org
+License: BSD
+Keywords: semantic version,versioning,version
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=2.7
+Description-Content-Type: text/x-rst
+Provides-Extra: dev
+Requires-Dist: Django (>=1.11) ; extra == 'dev'
+Requires-Dist: nose2 ; extra == 'dev'
+Requires-Dist: tox ; extra == 'dev'
+Requires-Dist: check-manifest ; extra == 'dev'
+Requires-Dist: coverage ; extra == 'dev'
+Requires-Dist: flake8 ; extra == 'dev'
+Requires-Dist: wheel ; extra == 'dev'
+Requires-Dist: zest.releaser[recommended] ; extra == 'dev'
+Requires-Dist: readme-renderer (<25.0) ; (python_version == "3.4") and extra == 'dev'
+Requires-Dist: colorama (<=0.4.1) ; (python_version == "3.4") and extra == 'dev'
+Provides-Extra: doc
+Requires-Dist: Sphinx ; extra == 'doc'
+Requires-Dist: sphinx-rtd-theme ; extra == 'doc'
+
+Introduction
+============
+
+This small python library provides a few tools to handle `SemVer`_ in Python.
+It follows strictly the 2.0.0 version of the SemVer scheme.
+
+.. image:: https://github.com/rbarrois/python-semanticversion/actions/workflows/test.yml/badge.svg
+    :target: https://github.com/rbarrois/python-semanticversion/actions/workflows/test.yml
+
+.. image:: https://img.shields.io/pypi/v/semantic_version.svg
+    :target: https://python-semanticversion.readthedocs.io/en/latest/changelog.html
+    :alt: Latest Version
+
+.. image:: https://img.shields.io/pypi/pyversions/semantic_version.svg
+    :target: https://pypi.python.org/pypi/semantic_version/
+    :alt: Supported Python versions
+
+.. image:: https://img.shields.io/pypi/wheel/semantic_version.svg
+    :target: https://pypi.python.org/pypi/semantic_version/
+    :alt: Wheel status
+
+.. image:: https://img.shields.io/pypi/l/semantic_version.svg
+    :target: https://pypi.python.org/pypi/semantic_version/
+    :alt: License
+
+Links
+-----
+
+- Package on `PyPI`_: https://pypi.org/project/semantic-version/
+- Doc on `ReadTheDocs `_: https://python-semanticversion.readthedocs.io/
+- Source on `GitHub `_: http://github.com/rbarrois/python-semanticversion/
+- Build on Github Actions: https://github.com/rbarrois/python-semanticversion/actions
+- Semantic Version specification: `SemVer`_
+
+
+Getting started
+===============
+
+Install the package from `PyPI`_, using pip:
+
+.. code-block:: sh
+
+    pip install semantic-version
+
+Or from GitHub:
+
+.. code-block:: sh
+
+    $ git clone git://github.com/rbarrois/python-semanticversion.git
+
+
+Import it in your code:
+
+
+.. code-block:: python
+
+    import semantic_version
+
+
+This module provides classes to handle semantic versions:
+
+- ``Version`` represents a version number (``0.1.1-alpha+build.2012-05-15``)
+- ``BaseSpec``-derived classes represent requirement specifications (``>=0.1.1,<0.3.0``):
+
+  - ``SimpleSpec`` describes a natural description syntax
+  - ``NpmSpec`` is used for NPM-style range descriptions.
+
+Versions
+--------
+
+Defining a ``Version`` is quite simple:
+
+
+.. code-block:: pycon
+
+    >>> import semantic_version
+    >>> v = semantic_version.Version('0.1.1')
+    >>> v.major
+    0
+    >>> v.minor
+    1
+    >>> v.patch
+    1
+    >>> v.prerelease
+    []
+    >>> v.build
+    []
+    >>> list(v)
+    [0, 1, 1, [], []]
+
+If the provided version string is invalid, a ``ValueError`` will be raised:
+
+.. code-block:: pycon
+
+    >>> semantic_version.Version('0.1')
+    Traceback (most recent call last):
+      File "", line 1, in 
+      File "/Users/rbarrois/dev/semantic_version/src/semantic_version/base.py", line 64, in __init__
+        major, minor, patch, prerelease, build = self.parse(version_string, partial)
+      File "/Users/rbarrois/dev/semantic_version/src/semantic_version/base.py", line 86, in parse
+        raise ValueError('Invalid version string: %r' % version_string)
+    ValueError: Invalid version string: '0.1'
+
+
+One may also create a ``Version`` with named components:
+
+.. code-block:: pycon
+
+    >>> semantic_version.Version(major=0, minor=1, patch=2)
+    Version('0.1.2')
+
+In that case, ``major``, ``minor`` and ``patch`` are mandatory, and must be integers.
+``prerelease`` and ``build``, if provided, must be tuples of strings:
+
+.. code-block:: pycon
+
+    >>> semantic_version.Version(major=0, minor=1, patch=2, prerelease=('alpha', '2'))
+    Version('0.1.2-alpha.2')
+
+
+Some user-supplied input might not match the semantic version scheme.
+For such cases, the ``Version.coerce`` method will try to convert any
+version-like string into a valid semver version:
+
+.. code-block:: pycon
+
+    >>> Version.coerce('0')
+    Version('0.0.0')
+    >>> Version.coerce('0.1.2.3.4')
+    Version('0.1.2+3.4')
+    >>> Version.coerce('0.1.2a3')
+    Version('0.1.2-a3')
+
+Working with versions
+"""""""""""""""""""""
+
+Obviously, versions can be compared:
+
+
+.. code-block:: pycon
+
+    >>> semantic_version.Version('0.1.1') < semantic_version.Version('0.1.2')
+    True
+    >>> semantic_version.Version('0.1.1') > semantic_version.Version('0.1.1-alpha')
+    True
+    >>> semantic_version.Version('0.1.1') <= semantic_version.Version('0.1.1-alpha')
+    False
+
+You can also get a new version that represents a bump in one of the version levels:
+
+.. code-block:: pycon
+
+    >>> v = semantic_version.Version('0.1.1+build')
+    >>> new_v = v.next_major()
+    >>> str(new_v)
+    '1.0.0'
+    >>> v = semantic_version.Version('1.1.1+build')
+    >>> new_v = v.next_minor()
+    >>> str(new_v)
+    '1.2.0'
+    >>> v = semantic_version.Version('1.1.1+build')
+    >>> new_v = v.next_patch()
+    >>> str(new_v)
+    '1.1.2'
+
+
+
+Requirement specification
+-------------------------
+
+python-semanticversion provides a couple of ways to describe a range of accepted
+versions:
+
+- The ``SimpleSpec`` class provides a simple, easily understood scheme --
+  somewhat inspired from PyPI range notations;
+- The ``NpmSpec`` class supports the whole NPM range specification scheme:
+
+  .. code-block:: pycon
+
+      >>> Version('0.1.2') in NpmSpec('0.1.0-alpha.2 .. 0.2.4')
+      True
+      >>> Version('0.1.2') in NpmSpec('>=0.1.1 <0.1.3 || 2.x')
+      True
+      >>> Version('2.3.4') in NpmSpec('>=0.1.1 <0.1.3 || 2.x')
+      True
+
+The ``SimpleSpec`` scheme
+"""""""""""""""""""""""""
+
+Basic usage is simply a comparator and a base version:
+
+.. code-block:: pycon
+
+    >>> s = SimpleSpec('>=0.1.1')  # At least 0.1.1
+    >>> s.match(Version('0.1.1'))
+    True
+    >>> s.match(Version('0.1.1-alpha1'))  # pre-release doesn't satisfy version spec
+    False
+    >>> s.match(Version('0.1.0'))
+    False
+
+Combining specifications can be expressed as follows:
+
+  .. code-block:: pycon
+
+      >>> SimpleSpec('>=0.1.1,<0.3.0')
+
+Simpler test syntax is also available using the ``in`` keyword:
+
+.. code-block:: pycon
+
+    >>> s = SimpleSpec('==0.1.1')
+    >>> Version('0.1.1+git7ccc72') in s  # build variants are equivalent to full versions
+    True
+    >>> Version('0.1.1-alpha1') in s     # pre-release variants don't match the full version.
+    False
+    >>> Version('0.1.2') in s
+    False
+
+
+Refer to the full documentation at
+https://python-semanticversion.readthedocs.io/en/latest/ for more details on the
+``SimpleSpec`` scheme.
+
+
+
+Using a specification
+"""""""""""""""""""""
+
+The ``SimpleSpec.filter`` method filters an iterable of ``Version``:
+
+.. code-block:: pycon
+
+    >>> s = SimpleSpec('>=0.1.0,<0.4.0')
+    >>> versions = (Version('0.%d.0' % i) for i in range(6))
+    >>> for v in s.filter(versions):
+    ...     print v
+    0.1.0
+    0.2.0
+    0.3.0
+
+It is also possible to select the 'best' version from such iterables:
+
+
+.. code-block:: pycon
+
+    >>> s = SimpleSpec('>=0.1.0,<0.4.0')
+    >>> versions = (Version('0.%d.0' % i) for i in range(6))
+    >>> s.select(versions)
+    Version('0.3.0')
+
+
+
+Contributing
+============
+
+In order to contribute to the source code:
+
+- Open an issue on `GitHub`_: https://github.com/rbarrois/python-semanticversion/issues
+- Fork the `repository `_
+  and submit a pull request on `GitHub`_
+- Or send me a patch (mailto:raphael.barrois+semver@polytechnique.org)
+
+When submitting patches or pull requests, you should respect the following rules:
+
+- Coding conventions are based on :pep:`8`
+- The whole test suite must pass after adding the changes
+- The test coverage for a new feature must be 100%
+- New features and methods should be documented in the ``reference`` section
+  and included in the ``changelog``
+- Include your name in the ``contributors`` section
+
+.. note:: All files should contain the following header::
+
+          # -*- encoding: utf-8 -*-
+          # Copyright (c) The python-semanticversion project
+
+.. _SemVer: http://semver.org/
+.. _PyPI: http://pypi.python.org/
+
+
diff --git a/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/RECORD b/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/RECORD
new file mode 100644
index 0000000000000000000000000000000000000000..74ce5f9d741bbafd87b5009e999f62d7e587f1fa
--- /dev/null
+++ b/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/RECORD
@@ -0,0 +1,12 @@
+semantic_version-2.10.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+semantic_version-2.10.0.dist-info/LICENSE,sha256=qF5-8vvGcNJnge1oRM0xp-itpl0hMo91oLAkAvquN-o,1317
+semantic_version-2.10.0.dist-info/METADATA,sha256=dBTUaaVpBGtqJGv7HxBaqxt6v9wyqVJT8VQy6jAVmqY,9687
+semantic_version-2.10.0.dist-info/RECORD,,
+semantic_version-2.10.0.dist-info/WHEEL,sha256=Z-nyYpwrcSqxfdux5Mbn_DQ525iP7J2DG3JgGvOYyTQ,110
+semantic_version-2.10.0.dist-info/top_level.txt,sha256=8WNFqF6w9sgwWGOgIxBq65HQ6zakOE7tCaGKz0inQlI,17
+semantic_version/__init__.py,sha256=6f5c-uBdxzZyyMmKQbijcQFQ_km9pCz5C5YWXW1B048,540
+semantic_version/__pycache__/__init__.cpython-310.pyc,,
+semantic_version/__pycache__/base.cpython-310.pyc,,
+semantic_version/__pycache__/django_fields.cpython-310.pyc,,
+semantic_version/base.py,sha256=WuLdqIg4g9zruxYsrPgoc8uaRjD8rwG6onWGTwylcQQ,48115
+semantic_version/django_fields.py,sha256=ctqNHIrK2BwhqEkXcUBbRvTGO7wZVdJU1ub-MTyqCt0,3510
diff --git a/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/WHEEL b/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/WHEEL
new file mode 100644
index 0000000000000000000000000000000000000000..01b8fc7d4a10cb8b4f1d21f11d3398d07d6b3478
--- /dev/null
+++ b/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/WHEEL
@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.36.2)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+
diff --git a/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/top_level.txt b/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/top_level.txt
new file mode 100644
index 0000000000000000000000000000000000000000..fe3c35c52dae41a319c987afce4b234e644f5a70
--- /dev/null
+++ b/lib/python3.1/site-packages/semantic_version-2.10.0.dist-info/top_level.txt
@@ -0,0 +1 @@
+semantic_version
diff --git a/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/INSTALLER b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/INSTALLER
new file mode 100644
index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68
--- /dev/null
+++ b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/LICENSE b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..b9077766e9b9bdcae49ea5c8fced750ed13ec8f7
--- /dev/null
+++ b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/LICENSE
@@ -0,0 +1,13 @@
+Copyright (c) 2018, Tzu-ping Chung 
+
+Permission to use, copy, modify, and distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
diff --git a/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/METADATA b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..52118f1e5c83bd7ef39196a749651fc87d176812
--- /dev/null
+++ b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/METADATA
@@ -0,0 +1,106 @@
+Metadata-Version: 2.1
+Name: shellingham
+Version: 1.5.4
+Summary: Tool to Detect Surrounding Shell
+Home-page: https://github.com/sarugaku/shellingham
+Author: Tzu-ping Chung
+Author-email: uranusjr@gmail.com
+License: ISC License
+Keywords: shell
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: ISC License (ISCL)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.7
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+
+=============================================
+Shellingham: Tool to Detect Surrounding Shell
+=============================================
+
+.. image:: https://img.shields.io/pypi/v/shellingham.svg
+    :target: https://pypi.org/project/shellingham/
+
+Shellingham detects what shell the current Python executable is running in.
+
+
+Usage
+=====
+
+.. code-block:: python
+
+    >>> import shellingham
+    >>> shellingham.detect_shell()
+    ('bash', '/bin/bash')
+
+``detect_shell`` pokes around the process's running environment to determine
+what shell it is run in. It returns a 2-tuple:
+
+* The shell name, always lowercased.
+* The command used to run the shell.
+
+``ShellDetectionFailure`` is raised if ``detect_shell`` fails to detect the
+surrounding shell.
+
+
+Notes
+=====
+
+* The shell name is always lowercased.
+* On Windows, the shell name is the name of the executable, minus the file
+  extension.
+
+
+Notes for Application Developers
+================================
+
+Remember, your application's user is not necessarily using a shell.
+Shellingham raises ``ShellDetectionFailure`` if there is no shell to detect,
+but *your application should almost never do this to your user*.
+
+A practical approach to this is to wrap ``detect_shell`` in a try block, and
+provide a sane default on failure
+
+.. code-block:: python
+
+    try:
+        shell = shellingham.detect_shell()
+    except shellingham.ShellDetectionFailure:
+        shell = provide_default()
+
+
+There are a few choices for you to choose from.
+
+* The POSIX standard mandates the environment variable ``SHELL`` to refer to
+  "the user's preferred command language interpreter". This is always available
+  (even if the user is not in an interactive session), and likely the correct
+  choice to launch an interactive sub-shell with.
+* A command ``sh`` is almost guaranteed to exist, likely at ``/bin/sh``, since
+  several POSIX tools rely on it. This should be suitable if you want to run a
+  (possibly non-interactive) script.
+* All versions of DOS and Windows have an environment variable ``COMSPEC``.
+  This can always be used to launch a usable command prompt (e.g. `cmd.exe` on
+  Windows).
+
+Here's a simple implementation to provide a default shell
+
+.. code-block:: python
+
+    import os
+
+    def provide_default():
+        if os.name == 'posix':
+            return os.environ['SHELL']
+        elif os.name == 'nt':
+            return os.environ['COMSPEC']
+        raise NotImplementedError(f'OS {os.name!r} support not available')
diff --git a/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/RECORD b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/RECORD
new file mode 100644
index 0000000000000000000000000000000000000000..a03e85926af1ddd60a801034a230b4812a4b9ffd
--- /dev/null
+++ b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/RECORD
@@ -0,0 +1,21 @@
+shellingham-1.5.4.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+shellingham-1.5.4.dist-info/LICENSE,sha256=84j9OMrRMRLB3A9mm76A5_hFQe26-3LzAw0sp2QsPJ0,751
+shellingham-1.5.4.dist-info/METADATA,sha256=GD2AIgo3STJieVc53TV8xbs_Sb05DMkZjVGA5UUaB_o,3461
+shellingham-1.5.4.dist-info/RECORD,,
+shellingham-1.5.4.dist-info/WHEEL,sha256=iYlv5fX357PQyRT2o6tw1bN-YcKFFHKqB_LwHO5wP-g,110
+shellingham-1.5.4.dist-info/top_level.txt,sha256=uKMQL5AKxPi4O9_Rbd838QeEs4ImpGQKNbEDZYqgBgk,12
+shellingham-1.5.4.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+shellingham/__init__.py,sha256=pAKXUPKUdwyErC0ZjS-5w-fRdSbmdcfvnpt_x1yWqtA,635
+shellingham/__pycache__/__init__.cpython-310.pyc,,
+shellingham/__pycache__/_core.cpython-310.pyc,,
+shellingham/__pycache__/nt.cpython-310.pyc,,
+shellingham/_core.py,sha256=v-CTr_7F7cJAtNnzpa1N_Hl8afkY5yiDA4joGmsUBu0,300
+shellingham/nt.py,sha256=m6J6SuwyqVVlxXT9Bc-9F_1x-T5u0gCFFrRAF2LIkeg,4516
+shellingham/posix/__init__.py,sha256=pB69qtvZJ_yIf48nl4-ZfS3wLwwuXuknXOZhBnC2T1o,3129
+shellingham/posix/__pycache__/__init__.cpython-310.pyc,,
+shellingham/posix/__pycache__/_core.cpython-310.pyc,,
+shellingham/posix/__pycache__/proc.cpython-310.pyc,,
+shellingham/posix/__pycache__/ps.cpython-310.pyc,,
+shellingham/posix/_core.py,sha256=_v18UaXbzr4muNhr3-mH1FdSdjZ_dOXQrtUyomIbKYQ,81
+shellingham/posix/proc.py,sha256=nSUxIuQSotvaDW76i0oTQAM9aZ9PXBLFAEktWljSKCo,2659
+shellingham/posix/ps.py,sha256=NGmDKCukhNp0lahwYCaMXphBYaVbhbiR9BtE0OkT8qU,1770
diff --git a/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/WHEEL b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/WHEEL
new file mode 100644
index 0000000000000000000000000000000000000000..c34f1162ef9a50c355df1261ef6194ffc1b39975
--- /dev/null
+++ b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/WHEEL
@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.41.2)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+
diff --git a/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/top_level.txt b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/top_level.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d4e44ce0299bb38463f8491ec8850910235c2709
--- /dev/null
+++ b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/top_level.txt
@@ -0,0 +1 @@
+shellingham
diff --git a/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/zip-safe b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/zip-safe
new file mode 100644
index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc
--- /dev/null
+++ b/lib/python3.1/site-packages/shellingham-1.5.4.dist-info/zip-safe
@@ -0,0 +1 @@
+
diff --git a/lib/python3.1/site-packages/six.py b/lib/python3.1/site-packages/six.py
new file mode 100644
index 0000000000000000000000000000000000000000..3de5969b1ad3b973342e5e88ee1770fa7c798152
--- /dev/null
+++ b/lib/python3.1/site-packages/six.py
@@ -0,0 +1,1003 @@
+# Copyright (c) 2010-2024 Benjamin Peterson
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Utilities for writing code that runs on Python 2 and 3"""
+
+from __future__ import absolute_import
+
+import functools
+import itertools
+import operator
+import sys
+import types
+
+__author__ = "Benjamin Peterson "
+__version__ = "1.17.0"
+
+
+# Useful for very coarse version differentiation.
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+PY34 = sys.version_info[0:2] >= (3, 4)
+
+if PY3:
+    string_types = str,
+    integer_types = int,
+    class_types = type,
+    text_type = str
+    binary_type = bytes
+
+    MAXSIZE = sys.maxsize
+else:
+    string_types = basestring,
+    integer_types = (int, long)
+    class_types = (type, types.ClassType)
+    text_type = unicode
+    binary_type = str
+
+    if sys.platform.startswith("java"):
+        # Jython always uses 32 bits.
+        MAXSIZE = int((1 << 31) - 1)
+    else:
+        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
+        class X(object):
+
+            def __len__(self):
+                return 1 << 31
+        try:
+            len(X())
+        except OverflowError:
+            # 32-bit
+            MAXSIZE = int((1 << 31) - 1)
+        else:
+            # 64-bit
+            MAXSIZE = int((1 << 63) - 1)
+        del X
+
+if PY34:
+    from importlib.util import spec_from_loader
+else:
+    spec_from_loader = None
+
+
+def _add_doc(func, doc):
+    """Add documentation to a function."""
+    func.__doc__ = doc
+
+
+def _import_module(name):
+    """Import module, returning the module after the last dot."""
+    __import__(name)
+    return sys.modules[name]
+
+
+class _LazyDescr(object):
+
+    def __init__(self, name):
+        self.name = name
+
+    def __get__(self, obj, tp):
+        result = self._resolve()
+        setattr(obj, self.name, result)  # Invokes __set__.
+        try:
+            # This is a bit ugly, but it avoids running this again by
+            # removing this descriptor.
+            delattr(obj.__class__, self.name)
+        except AttributeError:
+            pass
+        return result
+
+
+class MovedModule(_LazyDescr):
+
+    def __init__(self, name, old, new=None):
+        super(MovedModule, self).__init__(name)
+        if PY3:
+            if new is None:
+                new = name
+            self.mod = new
+        else:
+            self.mod = old
+
+    def _resolve(self):
+        return _import_module(self.mod)
+
+    def __getattr__(self, attr):
+        _module = self._resolve()
+        value = getattr(_module, attr)
+        setattr(self, attr, value)
+        return value
+
+
+class _LazyModule(types.ModuleType):
+
+    def __init__(self, name):
+        super(_LazyModule, self).__init__(name)
+        self.__doc__ = self.__class__.__doc__
+
+    def __dir__(self):
+        attrs = ["__doc__", "__name__"]
+        attrs += [attr.name for attr in self._moved_attributes]
+        return attrs
+
+    # Subclasses should override this
+    _moved_attributes = []
+
+
+class MovedAttribute(_LazyDescr):
+
+    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
+        super(MovedAttribute, self).__init__(name)
+        if PY3:
+            if new_mod is None:
+                new_mod = name
+            self.mod = new_mod
+            if new_attr is None:
+                if old_attr is None:
+                    new_attr = name
+                else:
+                    new_attr = old_attr
+            self.attr = new_attr
+        else:
+            self.mod = old_mod
+            if old_attr is None:
+                old_attr = name
+            self.attr = old_attr
+
+    def _resolve(self):
+        module = _import_module(self.mod)
+        return getattr(module, self.attr)
+
+
+class _SixMetaPathImporter(object):
+
+    """
+    A meta path importer to import six.moves and its submodules.
+
+    This class implements a PEP302 finder and loader. It should be compatible
+    with Python 2.5 and all existing versions of Python3
+    """
+
+    def __init__(self, six_module_name):
+        self.name = six_module_name
+        self.known_modules = {}
+
+    def _add_module(self, mod, *fullnames):
+        for fullname in fullnames:
+            self.known_modules[self.name + "." + fullname] = mod
+
+    def _get_module(self, fullname):
+        return self.known_modules[self.name + "." + fullname]
+
+    def find_module(self, fullname, path=None):
+        if fullname in self.known_modules:
+            return self
+        return None
+
+    def find_spec(self, fullname, path, target=None):
+        if fullname in self.known_modules:
+            return spec_from_loader(fullname, self)
+        return None
+
+    def __get_module(self, fullname):
+        try:
+            return self.known_modules[fullname]
+        except KeyError:
+            raise ImportError("This loader does not know module " + fullname)
+
+    def load_module(self, fullname):
+        try:
+            # in case of a reload
+            return sys.modules[fullname]
+        except KeyError:
+            pass
+        mod = self.__get_module(fullname)
+        if isinstance(mod, MovedModule):
+            mod = mod._resolve()
+        else:
+            mod.__loader__ = self
+        sys.modules[fullname] = mod
+        return mod
+
+    def is_package(self, fullname):
+        """
+        Return true, if the named module is a package.
+
+        We need this method to get correct spec objects with
+        Python 3.4 (see PEP451)
+        """
+        return hasattr(self.__get_module(fullname), "__path__")
+
+    def get_code(self, fullname):
+        """Return None
+
+        Required, if is_package is implemented"""
+        self.__get_module(fullname)  # eventually raises ImportError
+        return None
+    get_source = get_code  # same as get_code
+
+    def create_module(self, spec):
+        return self.load_module(spec.name)
+
+    def exec_module(self, module):
+        pass
+
+_importer = _SixMetaPathImporter(__name__)
+
+
+class _MovedItems(_LazyModule):
+
+    """Lazy loading of moved objects"""
+    __path__ = []  # mark as package
+
+
+_moved_attributes = [
+    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
+    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
+    MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
+    MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
+    MovedAttribute("intern", "__builtin__", "sys"),
+    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
+    MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
+    MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
+    MovedAttribute("getoutput", "commands", "subprocess"),
+    MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
+    MovedAttribute("reduce", "__builtin__", "functools"),
+    MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
+    MovedAttribute("StringIO", "StringIO", "io"),
+    MovedAttribute("UserDict", "UserDict", "collections", "IterableUserDict", "UserDict"),
+    MovedAttribute("UserList", "UserList", "collections"),
+    MovedAttribute("UserString", "UserString", "collections"),
+    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
+    MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
+    MovedModule("builtins", "__builtin__"),
+    MovedModule("configparser", "ConfigParser"),
+    MovedModule("collections_abc", "collections", "collections.abc" if sys.version_info >= (3, 3) else "collections"),
+    MovedModule("copyreg", "copy_reg"),
+    MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
+    MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"),
+    MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread" if sys.version_info < (3, 9) else "_thread"),
+    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
+    MovedModule("http_cookies", "Cookie", "http.cookies"),
+    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
+    MovedModule("html_parser", "HTMLParser", "html.parser"),
+    MovedModule("http_client", "httplib", "http.client"),
+    MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
+    MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"),
+    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
+    MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
+    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
+    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
+    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
+    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
+    MovedModule("cPickle", "cPickle", "pickle"),
+    MovedModule("queue", "Queue"),
+    MovedModule("reprlib", "repr"),
+    MovedModule("socketserver", "SocketServer"),
+    MovedModule("_thread", "thread", "_thread"),
+    MovedModule("tkinter", "Tkinter"),
+    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
+    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
+    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
+    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
+    MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
+    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
+    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
+    MovedModule("tkinter_colorchooser", "tkColorChooser",
+                "tkinter.colorchooser"),
+    MovedModule("tkinter_commondialog", "tkCommonDialog",
+                "tkinter.commondialog"),
+    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
+    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
+    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
+                "tkinter.simpledialog"),
+    MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
+    MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
+    MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
+    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
+    MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
+    MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
+]
+# Add windows specific modules.
+if sys.platform == "win32":
+    _moved_attributes += [
+        MovedModule("winreg", "_winreg"),
+    ]
+
+for attr in _moved_attributes:
+    setattr(_MovedItems, attr.name, attr)
+    if isinstance(attr, MovedModule):
+        _importer._add_module(attr, "moves." + attr.name)
+del attr
+
+_MovedItems._moved_attributes = _moved_attributes
+
+moves = _MovedItems(__name__ + ".moves")
+_importer._add_module(moves, "moves")
+
+
+class Module_six_moves_urllib_parse(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_parse"""
+
+
+_urllib_parse_moved_attributes = [
+    MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
+    MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
+    MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
+    MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
+    MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
+    MovedAttribute("urljoin", "urlparse", "urllib.parse"),
+    MovedAttribute("urlparse", "urlparse", "urllib.parse"),
+    MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
+    MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
+    MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
+    MovedAttribute("quote", "urllib", "urllib.parse"),
+    MovedAttribute("quote_plus", "urllib", "urllib.parse"),
+    MovedAttribute("unquote", "urllib", "urllib.parse"),
+    MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
+    MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"),
+    MovedAttribute("urlencode", "urllib", "urllib.parse"),
+    MovedAttribute("splitquery", "urllib", "urllib.parse"),
+    MovedAttribute("splittag", "urllib", "urllib.parse"),
+    MovedAttribute("splituser", "urllib", "urllib.parse"),
+    MovedAttribute("splitvalue", "urllib", "urllib.parse"),
+    MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_params", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_query", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
+]
+for attr in _urllib_parse_moved_attributes:
+    setattr(Module_six_moves_urllib_parse, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
+                      "moves.urllib_parse", "moves.urllib.parse")
+
+
+class Module_six_moves_urllib_error(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_error"""
+
+
+_urllib_error_moved_attributes = [
+    MovedAttribute("URLError", "urllib2", "urllib.error"),
+    MovedAttribute("HTTPError", "urllib2", "urllib.error"),
+    MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
+]
+for attr in _urllib_error_moved_attributes:
+    setattr(Module_six_moves_urllib_error, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
+                      "moves.urllib_error", "moves.urllib.error")
+
+
+class Module_six_moves_urllib_request(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_request"""
+
+
+_urllib_request_moved_attributes = [
+    MovedAttribute("urlopen", "urllib2", "urllib.request"),
+    MovedAttribute("install_opener", "urllib2", "urllib.request"),
+    MovedAttribute("build_opener", "urllib2", "urllib.request"),
+    MovedAttribute("pathname2url", "urllib", "urllib.request"),
+    MovedAttribute("url2pathname", "urllib", "urllib.request"),
+    MovedAttribute("getproxies", "urllib", "urllib.request"),
+    MovedAttribute("Request", "urllib2", "urllib.request"),
+    MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
+    MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
+    MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
+    MovedAttribute("FileHandler", "urllib2", "urllib.request"),
+    MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
+    MovedAttribute("urlretrieve", "urllib", "urllib.request"),
+    MovedAttribute("urlcleanup", "urllib", "urllib.request"),
+    MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
+    MovedAttribute("parse_http_list", "urllib2", "urllib.request"),
+    MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"),
+]
+if sys.version_info[:2] < (3, 14):
+    _urllib_request_moved_attributes.extend(
+        [
+            MovedAttribute("URLopener", "urllib", "urllib.request"),
+            MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
+        ]
+    )
+for attr in _urllib_request_moved_attributes:
+    setattr(Module_six_moves_urllib_request, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
+                      "moves.urllib_request", "moves.urllib.request")
+
+
+class Module_six_moves_urllib_response(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_response"""
+
+
+_urllib_response_moved_attributes = [
+    MovedAttribute("addbase", "urllib", "urllib.response"),
+    MovedAttribute("addclosehook", "urllib", "urllib.response"),
+    MovedAttribute("addinfo", "urllib", "urllib.response"),
+    MovedAttribute("addinfourl", "urllib", "urllib.response"),
+]
+for attr in _urllib_response_moved_attributes:
+    setattr(Module_six_moves_urllib_response, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
+                      "moves.urllib_response", "moves.urllib.response")
+
+
+class Module_six_moves_urllib_robotparser(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_robotparser"""
+
+
+_urllib_robotparser_moved_attributes = [
+    MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
+]
+for attr in _urllib_robotparser_moved_attributes:
+    setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
+                      "moves.urllib_robotparser", "moves.urllib.robotparser")
+
+
+class Module_six_moves_urllib(types.ModuleType):
+
+    """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
+    __path__ = []  # mark as package
+    parse = _importer._get_module("moves.urllib_parse")
+    error = _importer._get_module("moves.urllib_error")
+    request = _importer._get_module("moves.urllib_request")
+    response = _importer._get_module("moves.urllib_response")
+    robotparser = _importer._get_module("moves.urllib_robotparser")
+
+    def __dir__(self):
+        return ['parse', 'error', 'request', 'response', 'robotparser']
+
+_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
+                      "moves.urllib")
+
+
+def add_move(move):
+    """Add an item to six.moves."""
+    setattr(_MovedItems, move.name, move)
+
+
+def remove_move(name):
+    """Remove item from six.moves."""
+    try:
+        delattr(_MovedItems, name)
+    except AttributeError:
+        try:
+            del moves.__dict__[name]
+        except KeyError:
+            raise AttributeError("no such move, %r" % (name,))
+
+
+if PY3:
+    _meth_func = "__func__"
+    _meth_self = "__self__"
+
+    _func_closure = "__closure__"
+    _func_code = "__code__"
+    _func_defaults = "__defaults__"
+    _func_globals = "__globals__"
+else:
+    _meth_func = "im_func"
+    _meth_self = "im_self"
+
+    _func_closure = "func_closure"
+    _func_code = "func_code"
+    _func_defaults = "func_defaults"
+    _func_globals = "func_globals"
+
+
+try:
+    advance_iterator = next
+except NameError:
+    def advance_iterator(it):
+        return it.next()
+next = advance_iterator
+
+
+try:
+    callable = callable
+except NameError:
+    def callable(obj):
+        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
+
+
+if PY3:
+    def get_unbound_function(unbound):
+        return unbound
+
+    create_bound_method = types.MethodType
+
+    def create_unbound_method(func, cls):
+        return func
+
+    Iterator = object
+else:
+    def get_unbound_function(unbound):
+        return unbound.im_func
+
+    def create_bound_method(func, obj):
+        return types.MethodType(func, obj, obj.__class__)
+
+    def create_unbound_method(func, cls):
+        return types.MethodType(func, None, cls)
+
+    class Iterator(object):
+
+        def next(self):
+            return type(self).__next__(self)
+
+    callable = callable
+_add_doc(get_unbound_function,
+         """Get the function out of a possibly unbound function""")
+
+
+get_method_function = operator.attrgetter(_meth_func)
+get_method_self = operator.attrgetter(_meth_self)
+get_function_closure = operator.attrgetter(_func_closure)
+get_function_code = operator.attrgetter(_func_code)
+get_function_defaults = operator.attrgetter(_func_defaults)
+get_function_globals = operator.attrgetter(_func_globals)
+
+
+if PY3:
+    def iterkeys(d, **kw):
+        return iter(d.keys(**kw))
+
+    def itervalues(d, **kw):
+        return iter(d.values(**kw))
+
+    def iteritems(d, **kw):
+        return iter(d.items(**kw))
+
+    def iterlists(d, **kw):
+        return iter(d.lists(**kw))
+
+    viewkeys = operator.methodcaller("keys")
+
+    viewvalues = operator.methodcaller("values")
+
+    viewitems = operator.methodcaller("items")
+else:
+    def iterkeys(d, **kw):
+        return d.iterkeys(**kw)
+
+    def itervalues(d, **kw):
+        return d.itervalues(**kw)
+
+    def iteritems(d, **kw):
+        return d.iteritems(**kw)
+
+    def iterlists(d, **kw):
+        return d.iterlists(**kw)
+
+    viewkeys = operator.methodcaller("viewkeys")
+
+    viewvalues = operator.methodcaller("viewvalues")
+
+    viewitems = operator.methodcaller("viewitems")
+
+_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
+_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
+_add_doc(iteritems,
+         "Return an iterator over the (key, value) pairs of a dictionary.")
+_add_doc(iterlists,
+         "Return an iterator over the (key, [values]) pairs of a dictionary.")
+
+
+if PY3:
+    def b(s):
+        return s.encode("latin-1")
+
+    def u(s):
+        return s
+    unichr = chr
+    import struct
+    int2byte = struct.Struct(">B").pack
+    del struct
+    byte2int = operator.itemgetter(0)
+    indexbytes = operator.getitem
+    iterbytes = iter
+    import io
+    StringIO = io.StringIO
+    BytesIO = io.BytesIO
+    del io
+    _assertCountEqual = "assertCountEqual"
+    if sys.version_info[1] <= 1:
+        _assertRaisesRegex = "assertRaisesRegexp"
+        _assertRegex = "assertRegexpMatches"
+        _assertNotRegex = "assertNotRegexpMatches"
+    else:
+        _assertRaisesRegex = "assertRaisesRegex"
+        _assertRegex = "assertRegex"
+        _assertNotRegex = "assertNotRegex"
+else:
+    def b(s):
+        return s
+    # Workaround for standalone backslash
+
+    def u(s):
+        return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
+    unichr = unichr
+    int2byte = chr
+
+    def byte2int(bs):
+        return ord(bs[0])
+
+    def indexbytes(buf, i):
+        return ord(buf[i])
+    iterbytes = functools.partial(itertools.imap, ord)
+    import StringIO
+    StringIO = BytesIO = StringIO.StringIO
+    _assertCountEqual = "assertItemsEqual"
+    _assertRaisesRegex = "assertRaisesRegexp"
+    _assertRegex = "assertRegexpMatches"
+    _assertNotRegex = "assertNotRegexpMatches"
+_add_doc(b, """Byte literal""")
+_add_doc(u, """Text literal""")
+
+
+def assertCountEqual(self, *args, **kwargs):
+    return getattr(self, _assertCountEqual)(*args, **kwargs)
+
+
+def assertRaisesRegex(self, *args, **kwargs):
+    return getattr(self, _assertRaisesRegex)(*args, **kwargs)
+
+
+def assertRegex(self, *args, **kwargs):
+    return getattr(self, _assertRegex)(*args, **kwargs)
+
+
+def assertNotRegex(self, *args, **kwargs):
+    return getattr(self, _assertNotRegex)(*args, **kwargs)
+
+
+if PY3:
+    exec_ = getattr(moves.builtins, "exec")
+
+    def reraise(tp, value, tb=None):
+        try:
+            if value is None:
+                value = tp()
+            if value.__traceback__ is not tb:
+                raise value.with_traceback(tb)
+            raise value
+        finally:
+            value = None
+            tb = None
+
+else:
+    def exec_(_code_, _globs_=None, _locs_=None):
+        """Execute code in a namespace."""
+        if _globs_ is None:
+            frame = sys._getframe(1)
+            _globs_ = frame.f_globals
+            if _locs_ is None:
+                _locs_ = frame.f_locals
+            del frame
+        elif _locs_ is None:
+            _locs_ = _globs_
+        exec("""exec _code_ in _globs_, _locs_""")
+
+    exec_("""def reraise(tp, value, tb=None):
+    try:
+        raise tp, value, tb
+    finally:
+        tb = None
+""")
+
+
+if sys.version_info[:2] > (3,):
+    exec_("""def raise_from(value, from_value):
+    try:
+        raise value from from_value
+    finally:
+        value = None
+""")
+else:
+    def raise_from(value, from_value):
+        raise value
+
+
+print_ = getattr(moves.builtins, "print", None)
+if print_ is None:
+    def print_(*args, **kwargs):
+        """The new-style print function for Python 2.4 and 2.5."""
+        fp = kwargs.pop("file", sys.stdout)
+        if fp is None:
+            return
+
+        def write(data):
+            if not isinstance(data, basestring):
+                data = str(data)
+            # If the file has an encoding, encode unicode with it.
+            if (isinstance(fp, file) and
+                    isinstance(data, unicode) and
+                    fp.encoding is not None):
+                errors = getattr(fp, "errors", None)
+                if errors is None:
+                    errors = "strict"
+                data = data.encode(fp.encoding, errors)
+            fp.write(data)
+        want_unicode = False
+        sep = kwargs.pop("sep", None)
+        if sep is not None:
+            if isinstance(sep, unicode):
+                want_unicode = True
+            elif not isinstance(sep, str):
+                raise TypeError("sep must be None or a string")
+        end = kwargs.pop("end", None)
+        if end is not None:
+            if isinstance(end, unicode):
+                want_unicode = True
+            elif not isinstance(end, str):
+                raise TypeError("end must be None or a string")
+        if kwargs:
+            raise TypeError("invalid keyword arguments to print()")
+        if not want_unicode:
+            for arg in args:
+                if isinstance(arg, unicode):
+                    want_unicode = True
+                    break
+        if want_unicode:
+            newline = unicode("\n")
+            space = unicode(" ")
+        else:
+            newline = "\n"
+            space = " "
+        if sep is None:
+            sep = space
+        if end is None:
+            end = newline
+        for i, arg in enumerate(args):
+            if i:
+                write(sep)
+            write(arg)
+        write(end)
+if sys.version_info[:2] < (3, 3):
+    _print = print_
+
+    def print_(*args, **kwargs):
+        fp = kwargs.get("file", sys.stdout)
+        flush = kwargs.pop("flush", False)
+        _print(*args, **kwargs)
+        if flush and fp is not None:
+            fp.flush()
+
+_add_doc(reraise, """Reraise an exception.""")
+
+if sys.version_info[0:2] < (3, 4):
+    # This does exactly the same what the :func:`py3:functools.update_wrapper`
+    # function does on Python versions after 3.2. It sets the ``__wrapped__``
+    # attribute on ``wrapper`` object and it doesn't raise an error if any of
+    # the attributes mentioned in ``assigned`` and ``updated`` are missing on
+    # ``wrapped`` object.
+    def _update_wrapper(wrapper, wrapped,
+                        assigned=functools.WRAPPER_ASSIGNMENTS,
+                        updated=functools.WRAPPER_UPDATES):
+        for attr in assigned:
+            try:
+                value = getattr(wrapped, attr)
+            except AttributeError:
+                continue
+            else:
+                setattr(wrapper, attr, value)
+        for attr in updated:
+            getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
+        wrapper.__wrapped__ = wrapped
+        return wrapper
+    _update_wrapper.__doc__ = functools.update_wrapper.__doc__
+
+    def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
+              updated=functools.WRAPPER_UPDATES):
+        return functools.partial(_update_wrapper, wrapped=wrapped,
+                                 assigned=assigned, updated=updated)
+    wraps.__doc__ = functools.wraps.__doc__
+
+else:
+    wraps = functools.wraps
+
+
+def with_metaclass(meta, *bases):
+    """Create a base class with a metaclass."""
+    # This requires a bit of explanation: the basic idea is to make a dummy
+    # metaclass for one level of class instantiation that replaces itself with
+    # the actual metaclass.
+    class metaclass(type):
+
+        def __new__(cls, name, this_bases, d):
+            if sys.version_info[:2] >= (3, 7):
+                # This version introduced PEP 560 that requires a bit
+                # of extra care (we mimic what is done by __build_class__).
+                resolved_bases = types.resolve_bases(bases)
+                if resolved_bases is not bases:
+                    d['__orig_bases__'] = bases
+            else:
+                resolved_bases = bases
+            return meta(name, resolved_bases, d)
+
+        @classmethod
+        def __prepare__(cls, name, this_bases):
+            return meta.__prepare__(name, bases)
+    return type.__new__(metaclass, 'temporary_class', (), {})
+
+
+def add_metaclass(metaclass):
+    """Class decorator for creating a class with a metaclass."""
+    def wrapper(cls):
+        orig_vars = cls.__dict__.copy()
+        slots = orig_vars.get('__slots__')
+        if slots is not None:
+            if isinstance(slots, str):
+                slots = [slots]
+            for slots_var in slots:
+                orig_vars.pop(slots_var)
+        orig_vars.pop('__dict__', None)
+        orig_vars.pop('__weakref__', None)
+        if hasattr(cls, '__qualname__'):
+            orig_vars['__qualname__'] = cls.__qualname__
+        return metaclass(cls.__name__, cls.__bases__, orig_vars)
+    return wrapper
+
+
+def ensure_binary(s, encoding='utf-8', errors='strict'):
+    """Coerce **s** to six.binary_type.
+
+    For Python 2:
+      - `unicode` -> encoded to `str`
+      - `str` -> `str`
+
+    For Python 3:
+      - `str` -> encoded to `bytes`
+      - `bytes` -> `bytes`
+    """
+    if isinstance(s, binary_type):
+        return s
+    if isinstance(s, text_type):
+        return s.encode(encoding, errors)
+    raise TypeError("not expecting type '%s'" % type(s))
+
+
+def ensure_str(s, encoding='utf-8', errors='strict'):
+    """Coerce *s* to `str`.
+
+    For Python 2:
+      - `unicode` -> encoded to `str`
+      - `str` -> `str`
+
+    For Python 3:
+      - `str` -> `str`
+      - `bytes` -> decoded to `str`
+    """
+    # Optimization: Fast return for the common case.
+    if type(s) is str:
+        return s
+    if PY2 and isinstance(s, text_type):
+        return s.encode(encoding, errors)
+    elif PY3 and isinstance(s, binary_type):
+        return s.decode(encoding, errors)
+    elif not isinstance(s, (text_type, binary_type)):
+        raise TypeError("not expecting type '%s'" % type(s))
+    return s
+
+
+def ensure_text(s, encoding='utf-8', errors='strict'):
+    """Coerce *s* to six.text_type.
+
+    For Python 2:
+      - `unicode` -> `unicode`
+      - `str` -> `unicode`
+
+    For Python 3:
+      - `str` -> `str`
+      - `bytes` -> decoded to `str`
+    """
+    if isinstance(s, binary_type):
+        return s.decode(encoding, errors)
+    elif isinstance(s, text_type):
+        return s
+    else:
+        raise TypeError("not expecting type '%s'" % type(s))
+
+
+def python_2_unicode_compatible(klass):
+    """
+    A class decorator that defines __unicode__ and __str__ methods under Python 2.
+    Under Python 3 it does nothing.
+
+    To support Python 2 and 3 with a single code base, define a __str__ method
+    returning text and apply this decorator to the class.
+    """
+    if PY2:
+        if '__str__' not in klass.__dict__:
+            raise ValueError("@python_2_unicode_compatible cannot be applied "
+                             "to %s because it doesn't define __str__()." %
+                             klass.__name__)
+        klass.__unicode__ = klass.__str__
+        klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
+    return klass
+
+
+# Complete the moves implementation.
+# This code is at the end of this module to speed up module loading.
+# Turn this module into a package.
+__path__ = []  # required for PEP 302 and PEP 451
+__package__ = __name__  # see PEP 366 @ReservedAssignment
+if globals().get("__spec__") is not None:
+    __spec__.submodule_search_locations = []  # PEP 451 @UndefinedVariable
+# Remove other six meta path importers, since they cause problems. This can
+# happen if six is removed from sys.modules and then reloaded. (Setuptools does
+# this for some reason.)
+if sys.meta_path:
+    for i, importer in enumerate(sys.meta_path):
+        # Here's some real nastiness: Another "instance" of the six module might
+        # be floating around. Therefore, we can't use isinstance() to check for
+        # the six meta path importer, since the other six instance will have
+        # inserted an importer with different class.
+        if (type(importer).__name__ == "_SixMetaPathImporter" and
+                importer.name == __name__):
+            del sys.meta_path[i]
+            break
+    del i, importer
+# Finally, add the importer to the meta path import hook.
+sys.meta_path.append(_importer)
diff --git a/lib/python3.1/site-packages/socks.py b/lib/python3.1/site-packages/socks.py
new file mode 100644
index 0000000000000000000000000000000000000000..83b1435dfaa53bcc0806053bc2436ee7773262b8
--- /dev/null
+++ b/lib/python3.1/site-packages/socks.py
@@ -0,0 +1,847 @@
+from base64 import b64encode
+try:
+    from collections.abc import Callable
+except ImportError:
+    from collections import Callable
+from errno import EOPNOTSUPP, EINVAL, EAGAIN
+import functools
+from io import BytesIO
+import logging
+import os
+from os import SEEK_CUR
+import socket
+import struct
+import sys
+
+__version__ = "1.7.1"
+
+
+if os.name == "nt" and sys.version_info < (3, 0):
+    try:
+        import win_inet_pton
+    except ImportError:
+        raise ImportError(
+            "To run PySocks on Windows you must install win_inet_pton")
+
+log = logging.getLogger(__name__)
+
+PROXY_TYPE_SOCKS4 = SOCKS4 = 1
+PROXY_TYPE_SOCKS5 = SOCKS5 = 2
+PROXY_TYPE_HTTP = HTTP = 3
+
+PROXY_TYPES = {"SOCKS4": SOCKS4, "SOCKS5": SOCKS5, "HTTP": HTTP}
+PRINTABLE_PROXY_TYPES = dict(zip(PROXY_TYPES.values(), PROXY_TYPES.keys()))
+
+_orgsocket = _orig_socket = socket.socket
+
+
+def set_self_blocking(function):
+
+    @functools.wraps(function)
+    def wrapper(*args, **kwargs):
+        self = args[0]
+        try:
+            _is_blocking = self.gettimeout()
+            if _is_blocking == 0:
+                self.setblocking(True)
+            return function(*args, **kwargs)
+        except Exception as e:
+            raise
+        finally:
+            # set orgin blocking
+            if _is_blocking == 0:
+                self.setblocking(False)
+    return wrapper
+
+
+class ProxyError(IOError):
+    """Socket_err contains original socket.error exception."""
+    def __init__(self, msg, socket_err=None):
+        self.msg = msg
+        self.socket_err = socket_err
+
+        if socket_err:
+            self.msg += ": {}".format(socket_err)
+
+    def __str__(self):
+        return self.msg
+
+
+class GeneralProxyError(ProxyError):
+    pass
+
+
+class ProxyConnectionError(ProxyError):
+    pass
+
+
+class SOCKS5AuthError(ProxyError):
+    pass
+
+
+class SOCKS5Error(ProxyError):
+    pass
+
+
+class SOCKS4Error(ProxyError):
+    pass
+
+
+class HTTPError(ProxyError):
+    pass
+
+SOCKS4_ERRORS = {
+    0x5B: "Request rejected or failed",
+    0x5C: ("Request rejected because SOCKS server cannot connect to identd on"
+           " the client"),
+    0x5D: ("Request rejected because the client program and identd report"
+           " different user-ids")
+}
+
+SOCKS5_ERRORS = {
+    0x01: "General SOCKS server failure",
+    0x02: "Connection not allowed by ruleset",
+    0x03: "Network unreachable",
+    0x04: "Host unreachable",
+    0x05: "Connection refused",
+    0x06: "TTL expired",
+    0x07: "Command not supported, or protocol error",
+    0x08: "Address type not supported"
+}
+
+DEFAULT_PORTS = {SOCKS4: 1080, SOCKS5: 1080, HTTP: 8080}
+
+
+def set_default_proxy(proxy_type=None, addr=None, port=None, rdns=True,
+                      username=None, password=None):
+    """Sets a default proxy.
+
+    All further socksocket objects will use the default unless explicitly
+    changed. All parameters are as for socket.set_proxy()."""
+    socksocket.default_proxy = (proxy_type, addr, port, rdns,
+                                username.encode() if username else None,
+                                password.encode() if password else None)
+
+
+def setdefaultproxy(*args, **kwargs):
+    if "proxytype" in kwargs:
+        kwargs["proxy_type"] = kwargs.pop("proxytype")
+    return set_default_proxy(*args, **kwargs)
+
+
+def get_default_proxy():
+    """Returns the default proxy, set by set_default_proxy."""
+    return socksocket.default_proxy
+
+getdefaultproxy = get_default_proxy
+
+
+def wrap_module(module):
+    """Attempts to replace a module's socket library with a SOCKS socket.
+
+    Must set a default proxy using set_default_proxy(...) first. This will
+    only work on modules that import socket directly into the namespace;
+    most of the Python Standard Library falls into this category."""
+    if socksocket.default_proxy:
+        module.socket.socket = socksocket
+    else:
+        raise GeneralProxyError("No default proxy specified")
+
+wrapmodule = wrap_module
+
+
+def create_connection(dest_pair,
+                      timeout=None, source_address=None,
+                      proxy_type=None, proxy_addr=None,
+                      proxy_port=None, proxy_rdns=True,
+                      proxy_username=None, proxy_password=None,
+                      socket_options=None):
+    """create_connection(dest_pair, *[, timeout], **proxy_args) -> socket object
+
+    Like socket.create_connection(), but connects to proxy
+    before returning the socket object.
+
+    dest_pair - 2-tuple of (IP/hostname, port).
+    **proxy_args - Same args passed to socksocket.set_proxy() if present.
+    timeout - Optional socket timeout value, in seconds.
+    source_address - tuple (host, port) for the socket to bind to as its source
+    address before connecting (only for compatibility)
+    """
+    # Remove IPv6 brackets on the remote address and proxy address.
+    remote_host, remote_port = dest_pair
+    if remote_host.startswith("["):
+        remote_host = remote_host.strip("[]")
+    if proxy_addr and proxy_addr.startswith("["):
+        proxy_addr = proxy_addr.strip("[]")
+
+    err = None
+
+    # Allow the SOCKS proxy to be on IPv4 or IPv6 addresses.
+    for r in socket.getaddrinfo(proxy_addr, proxy_port, 0, socket.SOCK_STREAM):
+        family, socket_type, proto, canonname, sa = r
+        sock = None
+        try:
+            sock = socksocket(family, socket_type, proto)
+
+            if socket_options:
+                for opt in socket_options:
+                    sock.setsockopt(*opt)
+
+            if isinstance(timeout, (int, float)):
+                sock.settimeout(timeout)
+
+            if proxy_type:
+                sock.set_proxy(proxy_type, proxy_addr, proxy_port, proxy_rdns,
+                               proxy_username, proxy_password)
+            if source_address:
+                sock.bind(source_address)
+
+            sock.connect((remote_host, remote_port))
+            return sock
+
+        except (socket.error, ProxyError) as e:
+            err = e
+            if sock:
+                sock.close()
+                sock = None
+
+    if err:
+        raise err
+
+    raise socket.error("gai returned empty list.")
+
+
+class _BaseSocket(socket.socket):
+    """Allows Python 2 delegated methods such as send() to be overridden."""
+    def __init__(self, *pos, **kw):
+        _orig_socket.__init__(self, *pos, **kw)
+
+        self._savedmethods = dict()
+        for name in self._savenames:
+            self._savedmethods[name] = getattr(self, name)
+            delattr(self, name)  # Allows normal overriding mechanism to work
+
+    _savenames = list()
+
+
+def _makemethod(name):
+    return lambda self, *pos, **kw: self._savedmethods[name](*pos, **kw)
+for name in ("sendto", "send", "recvfrom", "recv"):
+    method = getattr(_BaseSocket, name, None)
+
+    # Determine if the method is not defined the usual way
+    # as a function in the class.
+    # Python 2 uses __slots__, so there are descriptors for each method,
+    # but they are not functions.
+    if not isinstance(method, Callable):
+        _BaseSocket._savenames.append(name)
+        setattr(_BaseSocket, name, _makemethod(name))
+
+
+class socksocket(_BaseSocket):
+    """socksocket([family[, type[, proto]]]) -> socket object
+
+    Open a SOCKS enabled socket. The parameters are the same as
+    those of the standard socket init. In order for SOCKS to work,
+    you must specify family=AF_INET and proto=0.
+    The "type" argument must be either SOCK_STREAM or SOCK_DGRAM.
+    """
+
+    default_proxy = None
+
+    def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM,
+                 proto=0, *args, **kwargs):
+        if type not in (socket.SOCK_STREAM, socket.SOCK_DGRAM):
+            msg = "Socket type must be stream or datagram, not {!r}"
+            raise ValueError(msg.format(type))
+
+        super(socksocket, self).__init__(family, type, proto, *args, **kwargs)
+        self._proxyconn = None  # TCP connection to keep UDP relay alive
+
+        if self.default_proxy:
+            self.proxy = self.default_proxy
+        else:
+            self.proxy = (None, None, None, None, None, None)
+        self.proxy_sockname = None
+        self.proxy_peername = None
+
+        self._timeout = None
+
+    def _readall(self, file, count):
+        """Receive EXACTLY the number of bytes requested from the file object.
+
+        Blocks until the required number of bytes have been received."""
+        data = b""
+        while len(data) < count:
+            d = file.read(count - len(data))
+            if not d:
+                raise GeneralProxyError("Connection closed unexpectedly")
+            data += d
+        return data
+
+    def settimeout(self, timeout):
+        self._timeout = timeout
+        try:
+            # test if we're connected, if so apply timeout
+            peer = self.get_proxy_peername()
+            super(socksocket, self).settimeout(self._timeout)
+        except socket.error:
+            pass
+
+    def gettimeout(self):
+        return self._timeout
+
+    def setblocking(self, v):
+        if v:
+            self.settimeout(None)
+        else:
+            self.settimeout(0.0)
+
+    def set_proxy(self, proxy_type=None, addr=None, port=None, rdns=True,
+                  username=None, password=None):
+        """ Sets the proxy to be used.
+
+        proxy_type -  The type of the proxy to be used. Three types
+                        are supported: PROXY_TYPE_SOCKS4 (including socks4a),
+                        PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP
+        addr -        The address of the server (IP or DNS).
+        port -        The port of the server. Defaults to 1080 for SOCKS
+                        servers and 8080 for HTTP proxy servers.
+        rdns -        Should DNS queries be performed on the remote side
+                       (rather than the local side). The default is True.
+                       Note: This has no effect with SOCKS4 servers.
+        username -    Username to authenticate with to the server.
+                       The default is no authentication.
+        password -    Password to authenticate with to the server.
+                       Only relevant when username is also provided."""
+        self.proxy = (proxy_type, addr, port, rdns,
+                      username.encode() if username else None,
+                      password.encode() if password else None)
+
+    def setproxy(self, *args, **kwargs):
+        if "proxytype" in kwargs:
+            kwargs["proxy_type"] = kwargs.pop("proxytype")
+        return self.set_proxy(*args, **kwargs)
+
+    def bind(self, *pos, **kw):
+        """Implements proxy connection for UDP sockets.
+
+        Happens during the bind() phase."""
+        (proxy_type, proxy_addr, proxy_port, rdns, username,
+         password) = self.proxy
+        if not proxy_type or self.type != socket.SOCK_DGRAM:
+            return _orig_socket.bind(self, *pos, **kw)
+
+        if self._proxyconn:
+            raise socket.error(EINVAL, "Socket already bound to an address")
+        if proxy_type != SOCKS5:
+            msg = "UDP only supported by SOCKS5 proxy type"
+            raise socket.error(EOPNOTSUPP, msg)
+        super(socksocket, self).bind(*pos, **kw)
+
+        # Need to specify actual local port because
+        # some relays drop packets if a port of zero is specified.
+        # Avoid specifying host address in case of NAT though.
+        _, port = self.getsockname()
+        dst = ("0", port)
+
+        self._proxyconn = _orig_socket()
+        proxy = self._proxy_addr()
+        self._proxyconn.connect(proxy)
+
+        UDP_ASSOCIATE = b"\x03"
+        _, relay = self._SOCKS5_request(self._proxyconn, UDP_ASSOCIATE, dst)
+
+        # The relay is most likely on the same host as the SOCKS proxy,
+        # but some proxies return a private IP address (10.x.y.z)
+        host, _ = proxy
+        _, port = relay
+        super(socksocket, self).connect((host, port))
+        super(socksocket, self).settimeout(self._timeout)
+        self.proxy_sockname = ("0.0.0.0", 0)  # Unknown
+
+    def sendto(self, bytes, *args, **kwargs):
+        if self.type != socket.SOCK_DGRAM:
+            return super(socksocket, self).sendto(bytes, *args, **kwargs)
+        if not self._proxyconn:
+            self.bind(("", 0))
+
+        address = args[-1]
+        flags = args[:-1]
+
+        header = BytesIO()
+        RSV = b"\x00\x00"
+        header.write(RSV)
+        STANDALONE = b"\x00"
+        header.write(STANDALONE)
+        self._write_SOCKS5_address(address, header)
+
+        sent = super(socksocket, self).send(header.getvalue() + bytes, *flags,
+                                            **kwargs)
+        return sent - header.tell()
+
+    def send(self, bytes, flags=0, **kwargs):
+        if self.type == socket.SOCK_DGRAM:
+            return self.sendto(bytes, flags, self.proxy_peername, **kwargs)
+        else:
+            return super(socksocket, self).send(bytes, flags, **kwargs)
+
+    def recvfrom(self, bufsize, flags=0):
+        if self.type != socket.SOCK_DGRAM:
+            return super(socksocket, self).recvfrom(bufsize, flags)
+        if not self._proxyconn:
+            self.bind(("", 0))
+
+        buf = BytesIO(super(socksocket, self).recv(bufsize + 1024, flags))
+        buf.seek(2, SEEK_CUR)
+        frag = buf.read(1)
+        if ord(frag):
+            raise NotImplementedError("Received UDP packet fragment")
+        fromhost, fromport = self._read_SOCKS5_address(buf)
+
+        if self.proxy_peername:
+            peerhost, peerport = self.proxy_peername
+            if fromhost != peerhost or peerport not in (0, fromport):
+                raise socket.error(EAGAIN, "Packet filtered")
+
+        return (buf.read(bufsize), (fromhost, fromport))
+
+    def recv(self, *pos, **kw):
+        bytes, _ = self.recvfrom(*pos, **kw)
+        return bytes
+
+    def close(self):
+        if self._proxyconn:
+            self._proxyconn.close()
+        return super(socksocket, self).close()
+
+    def get_proxy_sockname(self):
+        """Returns the bound IP address and port number at the proxy."""
+        return self.proxy_sockname
+
+    getproxysockname = get_proxy_sockname
+
+    def get_proxy_peername(self):
+        """
+        Returns the IP and port number of the proxy.
+        """
+        return self.getpeername()
+
+    getproxypeername = get_proxy_peername
+
+    def get_peername(self):
+        """Returns the IP address and port number of the destination machine.
+
+        Note: get_proxy_peername returns the proxy."""
+        return self.proxy_peername
+
+    getpeername = get_peername
+
+    def _negotiate_SOCKS5(self, *dest_addr):
+        """Negotiates a stream connection through a SOCKS5 server."""
+        CONNECT = b"\x01"
+        self.proxy_peername, self.proxy_sockname = self._SOCKS5_request(
+            self, CONNECT, dest_addr)
+
+    def _SOCKS5_request(self, conn, cmd, dst):
+        """
+        Send SOCKS5 request with given command (CMD field) and
+        address (DST field). Returns resolved DST address that was used.
+        """
+        proxy_type, addr, port, rdns, username, password = self.proxy
+
+        writer = conn.makefile("wb")
+        reader = conn.makefile("rb", 0)  # buffering=0 renamed in Python 3
+        try:
+            # First we'll send the authentication packages we support.
+            if username and password:
+                # The username/password details were supplied to the
+                # set_proxy method so we support the USERNAME/PASSWORD
+                # authentication (in addition to the standard none).
+                writer.write(b"\x05\x02\x00\x02")
+            else:
+                # No username/password were entered, therefore we
+                # only support connections with no authentication.
+                writer.write(b"\x05\x01\x00")
+
+            # We'll receive the server's response to determine which
+            # method was selected
+            writer.flush()
+            chosen_auth = self._readall(reader, 2)
+
+            if chosen_auth[0:1] != b"\x05":
+                # Note: string[i:i+1] is used because indexing of a bytestring
+                # via bytestring[i] yields an integer in Python 3
+                raise GeneralProxyError(
+                    "SOCKS5 proxy server sent invalid data")
+
+            # Check the chosen authentication method
+
+            if chosen_auth[1:2] == b"\x02":
+                # Okay, we need to perform a basic username/password
+                # authentication.
+                if not (username and password):
+                    # Although we said we don't support authentication, the
+                    # server may still request basic username/password
+                    # authentication
+                    raise SOCKS5AuthError("No username/password supplied. "
+                                          "Server requested username/password"
+                                          " authentication")
+
+                writer.write(b"\x01" + chr(len(username)).encode()
+                             + username
+                             + chr(len(password)).encode()
+                             + password)
+                writer.flush()
+                auth_status = self._readall(reader, 2)
+                if auth_status[0:1] != b"\x01":
+                    # Bad response
+                    raise GeneralProxyError(
+                        "SOCKS5 proxy server sent invalid data")
+                if auth_status[1:2] != b"\x00":
+                    # Authentication failed
+                    raise SOCKS5AuthError("SOCKS5 authentication failed")
+
+                # Otherwise, authentication succeeded
+
+            # No authentication is required if 0x00
+            elif chosen_auth[1:2] != b"\x00":
+                # Reaching here is always bad
+                if chosen_auth[1:2] == b"\xFF":
+                    raise SOCKS5AuthError(
+                        "All offered SOCKS5 authentication methods were"
+                        " rejected")
+                else:
+                    raise GeneralProxyError(
+                        "SOCKS5 proxy server sent invalid data")
+
+            # Now we can request the actual connection
+            writer.write(b"\x05" + cmd + b"\x00")
+            resolved = self._write_SOCKS5_address(dst, writer)
+            writer.flush()
+
+            # Get the response
+            resp = self._readall(reader, 3)
+            if resp[0:1] != b"\x05":
+                raise GeneralProxyError(
+                    "SOCKS5 proxy server sent invalid data")
+
+            status = ord(resp[1:2])
+            if status != 0x00:
+                # Connection failed: server returned an error
+                error = SOCKS5_ERRORS.get(status, "Unknown error")
+                raise SOCKS5Error("{:#04x}: {}".format(status, error))
+
+            # Get the bound address/port
+            bnd = self._read_SOCKS5_address(reader)
+
+            super(socksocket, self).settimeout(self._timeout)
+            return (resolved, bnd)
+        finally:
+            reader.close()
+            writer.close()
+
+    def _write_SOCKS5_address(self, addr, file):
+        """
+        Return the host and port packed for the SOCKS5 protocol,
+        and the resolved address as a tuple object.
+        """
+        host, port = addr
+        proxy_type, _, _, rdns, username, password = self.proxy
+        family_to_byte = {socket.AF_INET: b"\x01", socket.AF_INET6: b"\x04"}
+
+        # If the given destination address is an IP address, we'll
+        # use the IP address request even if remote resolving was specified.
+        # Detect whether the address is IPv4/6 directly.
+        for family in (socket.AF_INET, socket.AF_INET6):
+            try:
+                addr_bytes = socket.inet_pton(family, host)
+                file.write(family_to_byte[family] + addr_bytes)
+                host = socket.inet_ntop(family, addr_bytes)
+                file.write(struct.pack(">H", port))
+                return host, port
+            except socket.error:
+                continue
+
+        # Well it's not an IP number, so it's probably a DNS name.
+        if rdns:
+            # Resolve remotely
+            host_bytes = host.encode("idna")
+            file.write(b"\x03" + chr(len(host_bytes)).encode() + host_bytes)
+        else:
+            # Resolve locally
+            addresses = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
+                                           socket.SOCK_STREAM,
+                                           socket.IPPROTO_TCP,
+                                           socket.AI_ADDRCONFIG)
+            # We can't really work out what IP is reachable, so just pick the
+            # first.
+            target_addr = addresses[0]
+            family = target_addr[0]
+            host = target_addr[4][0]
+
+            addr_bytes = socket.inet_pton(family, host)
+            file.write(family_to_byte[family] + addr_bytes)
+            host = socket.inet_ntop(family, addr_bytes)
+        file.write(struct.pack(">H", port))
+        return host, port
+
+    def _read_SOCKS5_address(self, file):
+        atyp = self._readall(file, 1)
+        if atyp == b"\x01":
+            addr = socket.inet_ntoa(self._readall(file, 4))
+        elif atyp == b"\x03":
+            length = self._readall(file, 1)
+            addr = self._readall(file, ord(length))
+        elif atyp == b"\x04":
+            addr = socket.inet_ntop(socket.AF_INET6, self._readall(file, 16))
+        else:
+            raise GeneralProxyError("SOCKS5 proxy server sent invalid data")
+
+        port = struct.unpack(">H", self._readall(file, 2))[0]
+        return addr, port
+
+    def _negotiate_SOCKS4(self, dest_addr, dest_port):
+        """Negotiates a connection through a SOCKS4 server."""
+        proxy_type, addr, port, rdns, username, password = self.proxy
+
+        writer = self.makefile("wb")
+        reader = self.makefile("rb", 0)  # buffering=0 renamed in Python 3
+        try:
+            # Check if the destination address provided is an IP address
+            remote_resolve = False
+            try:
+                addr_bytes = socket.inet_aton(dest_addr)
+            except socket.error:
+                # It's a DNS name. Check where it should be resolved.
+                if rdns:
+                    addr_bytes = b"\x00\x00\x00\x01"
+                    remote_resolve = True
+                else:
+                    addr_bytes = socket.inet_aton(
+                        socket.gethostbyname(dest_addr))
+
+            # Construct the request packet
+            writer.write(struct.pack(">BBH", 0x04, 0x01, dest_port))
+            writer.write(addr_bytes)
+
+            # The username parameter is considered userid for SOCKS4
+            if username:
+                writer.write(username)
+            writer.write(b"\x00")
+
+            # DNS name if remote resolving is required
+            # NOTE: This is actually an extension to the SOCKS4 protocol
+            # called SOCKS4A and may not be supported in all cases.
+            if remote_resolve:
+                writer.write(dest_addr.encode("idna") + b"\x00")
+            writer.flush()
+
+            # Get the response from the server
+            resp = self._readall(reader, 8)
+            if resp[0:1] != b"\x00":
+                # Bad data
+                raise GeneralProxyError(
+                    "SOCKS4 proxy server sent invalid data")
+
+            status = ord(resp[1:2])
+            if status != 0x5A:
+                # Connection failed: server returned an error
+                error = SOCKS4_ERRORS.get(status, "Unknown error")
+                raise SOCKS4Error("{:#04x}: {}".format(status, error))
+
+            # Get the bound address/port
+            self.proxy_sockname = (socket.inet_ntoa(resp[4:]),
+                                   struct.unpack(">H", resp[2:4])[0])
+            if remote_resolve:
+                self.proxy_peername = socket.inet_ntoa(addr_bytes), dest_port
+            else:
+                self.proxy_peername = dest_addr, dest_port
+        finally:
+            reader.close()
+            writer.close()
+
+    def _negotiate_HTTP(self, dest_addr, dest_port):
+        """Negotiates a connection through an HTTP server.
+
+        NOTE: This currently only supports HTTP CONNECT-style proxies."""
+        proxy_type, addr, port, rdns, username, password = self.proxy
+
+        # If we need to resolve locally, we do this now
+        addr = dest_addr if rdns else socket.gethostbyname(dest_addr)
+
+        http_headers = [
+            (b"CONNECT " + addr.encode("idna") + b":"
+             + str(dest_port).encode() + b" HTTP/1.1"),
+            b"Host: " + dest_addr.encode("idna")
+        ]
+
+        if username and password:
+            http_headers.append(b"Proxy-Authorization: basic "
+                                + b64encode(username + b":" + password))
+
+        http_headers.append(b"\r\n")
+
+        self.sendall(b"\r\n".join(http_headers))
+
+        # We just need the first line to check if the connection was successful
+        fobj = self.makefile()
+        status_line = fobj.readline()
+        fobj.close()
+
+        if not status_line:
+            raise GeneralProxyError("Connection closed unexpectedly")
+
+        try:
+            proto, status_code, status_msg = status_line.split(" ", 2)
+        except ValueError:
+            raise GeneralProxyError("HTTP proxy server sent invalid response")
+
+        if not proto.startswith("HTTP/"):
+            raise GeneralProxyError(
+                "Proxy server does not appear to be an HTTP proxy")
+
+        try:
+            status_code = int(status_code)
+        except ValueError:
+            raise HTTPError(
+                "HTTP proxy server did not return a valid HTTP status")
+
+        if status_code != 200:
+            error = "{}: {}".format(status_code, status_msg)
+            if status_code in (400, 403, 405):
+                # It's likely that the HTTP proxy server does not support the
+                # CONNECT tunneling method
+                error += ("\n[*] Note: The HTTP proxy server may not be"
+                          " supported by PySocks (must be a CONNECT tunnel"
+                          " proxy)")
+            raise HTTPError(error)
+
+        self.proxy_sockname = (b"0.0.0.0", 0)
+        self.proxy_peername = addr, dest_port
+
+    _proxy_negotiators = {
+                           SOCKS4: _negotiate_SOCKS4,
+                           SOCKS5: _negotiate_SOCKS5,
+                           HTTP: _negotiate_HTTP
+                         }
+
+    @set_self_blocking
+    def connect(self, dest_pair, catch_errors=None):
+        """
+        Connects to the specified destination through a proxy.
+        Uses the same API as socket's connect().
+        To select the proxy server, use set_proxy().
+
+        dest_pair - 2-tuple of (IP/hostname, port).
+        """
+        if len(dest_pair) != 2 or dest_pair[0].startswith("["):
+            # Probably IPv6, not supported -- raise an error, and hope
+            # Happy Eyeballs (RFC6555) makes sure at least the IPv4
+            # connection works...
+            raise socket.error("PySocks doesn't support IPv6: %s"
+                               % str(dest_pair))
+
+        dest_addr, dest_port = dest_pair
+
+        if self.type == socket.SOCK_DGRAM:
+            if not self._proxyconn:
+                self.bind(("", 0))
+            dest_addr = socket.gethostbyname(dest_addr)
+
+            # If the host address is INADDR_ANY or similar, reset the peer
+            # address so that packets are received from any peer
+            if dest_addr == "0.0.0.0" and not dest_port:
+                self.proxy_peername = None
+            else:
+                self.proxy_peername = (dest_addr, dest_port)
+            return
+
+        (proxy_type, proxy_addr, proxy_port, rdns, username,
+         password) = self.proxy
+
+        # Do a minimal input check first
+        if (not isinstance(dest_pair, (list, tuple))
+                or len(dest_pair) != 2
+                or not dest_addr
+                or not isinstance(dest_port, int)):
+            # Inputs failed, raise an error
+            raise GeneralProxyError(
+                "Invalid destination-connection (host, port) pair")
+
+        # We set the timeout here so that we don't hang in connection or during
+        # negotiation.
+        super(socksocket, self).settimeout(self._timeout)
+
+        if proxy_type is None:
+            # Treat like regular socket object
+            self.proxy_peername = dest_pair
+            super(socksocket, self).settimeout(self._timeout)
+            super(socksocket, self).connect((dest_addr, dest_port))
+            return
+
+        proxy_addr = self._proxy_addr()
+
+        try:
+            # Initial connection to proxy server.
+            super(socksocket, self).connect(proxy_addr)
+
+        except socket.error as error:
+            # Error while connecting to proxy
+            self.close()
+            if not catch_errors:
+                proxy_addr, proxy_port = proxy_addr
+                proxy_server = "{}:{}".format(proxy_addr, proxy_port)
+                printable_type = PRINTABLE_PROXY_TYPES[proxy_type]
+
+                msg = "Error connecting to {} proxy {}".format(printable_type,
+                                                                    proxy_server)
+                log.debug("%s due to: %s", msg, error)
+                raise ProxyConnectionError(msg, error)
+            else:
+                raise error
+
+        else:
+            # Connected to proxy server, now negotiate
+            try:
+                # Calls negotiate_{SOCKS4, SOCKS5, HTTP}
+                negotiate = self._proxy_negotiators[proxy_type]
+                negotiate(self, dest_addr, dest_port)
+            except socket.error as error:
+                if not catch_errors:
+                    # Wrap socket errors
+                    self.close()
+                    raise GeneralProxyError("Socket error", error)
+                else:
+                    raise error
+            except ProxyError:
+                # Protocol error while negotiating with proxy
+                self.close()
+                raise
+                
+    @set_self_blocking
+    def connect_ex(self, dest_pair):
+        """ https://docs.python.org/3/library/socket.html#socket.socket.connect_ex
+        Like connect(address), but return an error indicator instead of raising an exception for errors returned by the C-level connect() call (other problems, such as "host not found" can still raise exceptions).
+        """
+        try:
+            self.connect(dest_pair, catch_errors=True)
+            return 0
+        except OSError as e:
+            # If the error is numeric (socket errors are numeric), then return number as 
+            # connect_ex expects. Otherwise raise the error again (socket timeout for example)
+            if e.errno:
+                return e.errno
+            else:
+                raise
+
+    def _proxy_addr(self):
+        """
+        Return proxy address to connect to as tuple object
+        """
+        (proxy_type, proxy_addr, proxy_port, rdns, username,
+         password) = self.proxy
+        proxy_port = proxy_port or DEFAULT_PORTS.get(proxy_type)
+        if not proxy_port:
+            raise GeneralProxyError("Invalid proxy type")
+        return proxy_addr, proxy_port
diff --git a/lib/python3.1/site-packages/sockshandler.py b/lib/python3.1/site-packages/sockshandler.py
new file mode 100644
index 0000000000000000000000000000000000000000..6a2ed81ceef43bae49f256824374bd8ae2576795
--- /dev/null
+++ b/lib/python3.1/site-packages/sockshandler.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+"""
+SocksiPy + urllib2 handler
+
+version: 0.3
+author: e
+
+This module provides a Handler which you can use with urllib2 to allow it to tunnel your connection through a socks.sockssocket socket, with out monkey patching the original socket...
+"""
+import socket
+import ssl
+
+try:
+    import urllib2
+    import httplib
+except ImportError: # Python 3
+    import urllib.request as urllib2
+    import http.client as httplib
+
+import socks # $ pip install PySocks
+
+def merge_dict(a, b):
+    d = a.copy()
+    d.update(b)
+    return d
+
+def is_ip(s):
+    try:
+        if ':' in s:
+            socket.inet_pton(socket.AF_INET6, s)
+        elif '.' in s:
+            socket.inet_aton(s)
+        else:
+            return False
+    except:
+        return False
+    else:
+        return True
+
+socks4_no_rdns = set()
+
+class SocksiPyConnection(httplib.HTTPConnection):
+    def __init__(self, proxytype, proxyaddr, proxyport=None, rdns=True, username=None, password=None, *args, **kwargs):
+        self.proxyargs = (proxytype, proxyaddr, proxyport, rdns, username, password)
+        httplib.HTTPConnection.__init__(self, *args, **kwargs)
+
+    def connect(self):
+        (proxytype, proxyaddr, proxyport, rdns, username, password) = self.proxyargs
+        rdns = rdns and proxyaddr not in socks4_no_rdns
+        while True:
+            try:
+                sock = socks.create_connection(
+                    (self.host, self.port), self.timeout, None,
+                    proxytype, proxyaddr, proxyport, rdns, username, password,
+                    ((socket.IPPROTO_TCP, socket.TCP_NODELAY, 1),))
+                break
+            except socks.SOCKS4Error as e:
+                if rdns and "0x5b" in str(e) and not is_ip(self.host):
+                    # Maybe a SOCKS4 server that doesn't support remote resolving
+                    # Let's try again
+                    rdns = False
+                    socks4_no_rdns.add(proxyaddr)
+                else:
+                    raise
+        self.sock = sock
+
+class SocksiPyConnectionS(httplib.HTTPSConnection):
+    def __init__(self, proxytype, proxyaddr, proxyport=None, rdns=True, username=None, password=None, *args, **kwargs):
+        self.proxyargs = (proxytype, proxyaddr, proxyport, rdns, username, password)
+        httplib.HTTPSConnection.__init__(self, *args, **kwargs)
+
+    def connect(self):
+        SocksiPyConnection.connect(self)
+        self.sock = self._context.wrap_socket(self.sock, server_hostname=self.host)
+        if not self._context.check_hostname and self._check_hostname:
+            try:
+                ssl.match_hostname(self.sock.getpeercert(), self.host)
+            except Exception:
+                self.sock.shutdown(socket.SHUT_RDWR)
+                self.sock.close()
+                raise
+
+class SocksiPyHandler(urllib2.HTTPHandler, urllib2.HTTPSHandler):
+    def __init__(self, *args, **kwargs):
+        self.args = args
+        self.kw = kwargs
+        urllib2.HTTPHandler.__init__(self)
+
+    def http_open(self, req):
+        def build(host, port=None, timeout=0, **kwargs):
+            kw = merge_dict(self.kw, kwargs)
+            conn = SocksiPyConnection(*self.args, host=host, port=port, timeout=timeout, **kw)
+            return conn
+        return self.do_open(build, req)
+
+    def https_open(self, req):
+        def build(host, port=None, timeout=0, **kwargs):
+            kw = merge_dict(self.kw, kwargs)
+            conn = SocksiPyConnectionS(*self.args, host=host, port=port, timeout=timeout, **kw)
+            return conn
+        return self.do_open(build, req)
+
+if __name__ == "__main__":
+    import sys
+    try:
+        port = int(sys.argv[1])
+    except (ValueError, IndexError):
+        port = 9050
+    opener = urllib2.build_opener(SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, "localhost", port))
+    print("HTTP: " + opener.open("http://httpbin.org/ip").read().decode())
+    print("HTTPS: " + opener.open("https://httpbin.org/ip").read().decode())
diff --git a/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/INSTALLER b/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/INSTALLER
new file mode 100644
index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68
--- /dev/null
+++ b/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/METADATA b/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..2410f166f31a21271d0728d19ba083cd37d24206
--- /dev/null
+++ b/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/METADATA
@@ -0,0 +1,400 @@
+Metadata-Version: 2.4
+Name: threadpoolctl
+Version: 3.6.0
+Summary: threadpoolctl
+Home-page: https://github.com/joblib/threadpoolctl
+Author: Thomas Moreau
+Author-email: thomas.moreau.2010@gmail.com
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License: BSD-3-Clause
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+License-File: LICENSE
+
+# Thread-pool Controls [![Build Status](https://github.com/joblib/threadpoolctl/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/joblib/threadpoolctl/actions?query=branch%3Amaster) [![codecov](https://codecov.io/gh/joblib/threadpoolctl/branch/master/graph/badge.svg)](https://codecov.io/gh/joblib/threadpoolctl)
+
+Python helpers to limit the number of threads used in the
+threadpool-backed of common native libraries used for scientific
+computing and data science (e.g. BLAS and OpenMP).
+
+Fine control of the underlying thread-pool size can be useful in
+workloads that involve nested parallelism so as to mitigate
+oversubscription issues.
+
+## Installation
+
+- For users, install the last published version from PyPI:
+
+  ```bash
+  pip install threadpoolctl
+  ```
+
+- For contributors, install from the source repository in developer
+  mode:
+
+  ```bash
+  pip install -r dev-requirements.txt
+  flit install --symlink
+  ```
+
+  then you run the tests with pytest:
+
+  ```bash
+  pytest
+  ```
+
+## Usage
+
+### Command Line Interface
+
+Get a JSON description of thread-pools initialized when importing python
+packages such as numpy or scipy for instance:
+
+```
+python -m threadpoolctl -i numpy scipy.linalg
+[
+  {
+    "filepath": "/home/ogrisel/miniconda3/envs/tmp/lib/libmkl_rt.so",
+    "prefix": "libmkl_rt",
+    "user_api": "blas",
+    "internal_api": "mkl",
+    "version": "2019.0.4",
+    "num_threads": 2,
+    "threading_layer": "intel"
+  },
+  {
+    "filepath": "/home/ogrisel/miniconda3/envs/tmp/lib/libiomp5.so",
+    "prefix": "libiomp",
+    "user_api": "openmp",
+    "internal_api": "openmp",
+    "version": null,
+    "num_threads": 4
+  }
+]
+```
+
+The JSON information is written on STDOUT. If some of the packages are missing,
+a warning message is displayed on STDERR.
+
+### Python Runtime Programmatic Introspection
+
+Introspect the current state of the threadpool-enabled runtime libraries
+that are loaded when importing Python packages:
+
+```python
+>>> from threadpoolctl import threadpool_info
+>>> from pprint import pprint
+>>> pprint(threadpool_info())
+[]
+
+>>> import numpy
+>>> pprint(threadpool_info())
+[{'filepath': '/home/ogrisel/miniconda3/envs/tmp/lib/libmkl_rt.so',
+  'internal_api': 'mkl',
+  'num_threads': 2,
+  'prefix': 'libmkl_rt',
+  'threading_layer': 'intel',
+  'user_api': 'blas',
+  'version': '2019.0.4'},
+ {'filepath': '/home/ogrisel/miniconda3/envs/tmp/lib/libiomp5.so',
+  'internal_api': 'openmp',
+  'num_threads': 4,
+  'prefix': 'libiomp',
+  'user_api': 'openmp',
+  'version': None}]
+
+>>> import xgboost
+>>> pprint(threadpool_info())
+[{'filepath': '/home/ogrisel/miniconda3/envs/tmp/lib/libmkl_rt.so',
+  'internal_api': 'mkl',
+  'num_threads': 2,
+  'prefix': 'libmkl_rt',
+  'threading_layer': 'intel',
+  'user_api': 'blas',
+  'version': '2019.0.4'},
+ {'filepath': '/home/ogrisel/miniconda3/envs/tmp/lib/libiomp5.so',
+  'internal_api': 'openmp',
+  'num_threads': 4,
+  'prefix': 'libiomp',
+  'user_api': 'openmp',
+  'version': None},
+ {'filepath': '/home/ogrisel/miniconda3/envs/tmp/lib/libgomp.so.1.0.0',
+  'internal_api': 'openmp',
+  'num_threads': 4,
+  'prefix': 'libgomp',
+  'user_api': 'openmp',
+  'version': None}]
+```
+
+In the above example, `numpy` was installed from the default anaconda channel and comes
+with MKL and its Intel OpenMP (`libiomp5`) implementation while `xgboost` was installed
+from pypi.org and links against GNU OpenMP (`libgomp`) so both OpenMP runtimes are
+loaded in the same Python program.
+
+The state of these libraries is also accessible through the object oriented API:
+
+```python
+>>> from threadpoolctl import ThreadpoolController, threadpool_info
+>>> from pprint import pprint
+>>> import numpy
+>>> controller = ThreadpoolController()
+>>> pprint(controller.info())
+[{'architecture': 'Haswell',
+  'filepath': '/home/jeremie/miniconda/envs/dev/lib/libopenblasp-r0.3.17.so',
+  'internal_api': 'openblas',
+  'num_threads': 4,
+  'prefix': 'libopenblas',
+  'threading_layer': 'pthreads',
+  'user_api': 'blas',
+  'version': '0.3.17'}]
+
+>>> controller.info() == threadpool_info()
+True
+```
+
+### Setting the Maximum Size of Thread-Pools
+
+Control the number of threads used by the underlying runtime libraries
+in specific sections of your Python program:
+
+```python
+>>> from threadpoolctl import threadpool_limits
+>>> import numpy as np
+
+>>> with threadpool_limits(limits=1, user_api='blas'):
+...     # In this block, calls to blas implementation (like openblas or MKL)
+...     # will be limited to use only one thread. They can thus be used jointly
+...     # with thread-parallelism.
+...     a = np.random.randn(1000, 1000)
+...     a_squared = a @ a
+```
+
+The threadpools can also be controlled via the object oriented API, which is especially
+useful to avoid searching through all the loaded shared libraries each time. It will
+however not act on libraries loaded after the instantiation of the
+`ThreadpoolController`:
+
+```python
+>>> from threadpoolctl import ThreadpoolController
+>>> import numpy as np
+>>> controller = ThreadpoolController()
+
+>>> with controller.limit(limits=1, user_api='blas'):
+...     a = np.random.randn(1000, 1000)
+...     a_squared = a @ a
+```
+
+### Restricting the limits to the scope of a function
+
+`threadpool_limits` and `ThreadpoolController` can also be used as decorators to set
+the maximum number of threads used by the supported libraries at a function level. The
+decorators are accessible through their `wrap` method:
+
+```python
+>>> from threadpoolctl import ThreadpoolController, threadpool_limits
+>>> import numpy as np
+>>> controller = ThreadpoolController()
+
+>>> @controller.wrap(limits=1, user_api='blas')
+... # or @threadpool_limits.wrap(limits=1, user_api='blas')
+... def my_func():
+...     # Inside this function, calls to blas implementation (like openblas or MKL)
+...     # will be limited to use only one thread.
+...     a = np.random.randn(1000, 1000)
+...     a_squared = a @ a
+...
+```
+
+### Switching the FlexiBLAS backend
+
+`FlexiBLAS` is a BLAS wrapper for which the BLAS backend can be switched at runtime.
+`threadpoolctl` exposes python bindings for this feature. Here's an example but note
+that this part of the API is experimental and subject to change without deprecation:
+
+```python
+>>> from threadpoolctl import ThreadpoolController
+>>> import numpy as np
+>>> controller = ThreadpoolController()
+
+>>> controller.info()
+[{'user_api': 'blas',
+  'internal_api': 'flexiblas',
+  'num_threads': 1,
+  'prefix': 'libflexiblas',
+  'filepath': '/usr/local/lib/libflexiblas.so.3.3',
+  'version': '3.3.1',
+  'available_backends': ['NETLIB', 'OPENBLASPTHREAD', 'ATLAS'],
+  'loaded_backends': ['NETLIB'],
+  'current_backend': 'NETLIB'}]
+
+# Retrieve the flexiblas controller
+>>> flexiblas_ct = controller.select(internal_api="flexiblas").lib_controllers[0]
+
+# Switch the backend with one predefined at build time (listed in "available_backends")
+>>> flexiblas_ct.switch_backend("OPENBLASPTHREAD")
+>>> controller.info()
+[{'user_api': 'blas',
+  'internal_api': 'flexiblas',
+  'num_threads': 4,
+  'prefix': 'libflexiblas',
+  'filepath': '/usr/local/lib/libflexiblas.so.3.3',
+  'version': '3.3.1',
+  'available_backends': ['NETLIB', 'OPENBLASPTHREAD', 'ATLAS'],
+  'loaded_backends': ['NETLIB', 'OPENBLASPTHREAD'],
+  'current_backend': 'OPENBLASPTHREAD'},
+ {'user_api': 'blas',
+  'internal_api': 'openblas',
+  'num_threads': 4,
+  'prefix': 'libopenblas',
+  'filepath': '/usr/lib/x86_64-linux-gnu/openblas-pthread/libopenblasp-r0.3.8.so',
+  'version': '0.3.8',
+  'threading_layer': 'pthreads',
+  'architecture': 'Haswell'}]
+
+# It's also possible to directly give the path to a shared library
+>>> flexiblas_controller.switch_backend("/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so")
+>>> controller.info()
+[{'user_api': 'blas',
+  'internal_api': 'flexiblas',
+  'num_threads': 2,
+  'prefix': 'libflexiblas',
+  'filepath': '/usr/local/lib/libflexiblas.so.3.3',
+  'version': '3.3.1',
+  'available_backends': ['NETLIB', 'OPENBLASPTHREAD', 'ATLAS'],
+  'loaded_backends': ['NETLIB',
+   'OPENBLASPTHREAD',
+   '/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so'],
+  'current_backend': '/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so'},
+ {'user_api': 'openmp',
+  'internal_api': 'openmp',
+  'num_threads': 4,
+  'prefix': 'libomp',
+  'filepath': '/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libomp.so',
+  'version': None},
+ {'user_api': 'blas',
+  'internal_api': 'openblas',
+  'num_threads': 4,
+  'prefix': 'libopenblas',
+  'filepath': '/usr/lib/x86_64-linux-gnu/openblas-pthread/libopenblasp-r0.3.8.so',
+  'version': '0.3.8',
+  'threading_layer': 'pthreads',
+  'architecture': 'Haswell'},
+ {'user_api': 'blas',
+  'internal_api': 'mkl',
+  'num_threads': 2,
+  'prefix': 'libmkl_rt',
+  'filepath': '/home/jeremie/miniforge/envs/flexiblas_threadpoolctl/lib/libmkl_rt.so.2',
+  'version': '2024.0-Product',
+  'threading_layer': 'gnu'}]
+```
+
+You can observe that the previously linked OpenBLAS shared object stays loaded by
+the Python program indefinitely, but FlexiBLAS itself no longer delegates BLAS calls
+to OpenBLAS as indicated by the `current_backend` attribute.
+### Writing a custom library controller
+
+Currently, `threadpoolctl` has support for `OpenMP` and the main `BLAS` libraries.
+However it can also be used to control the threadpool of other native libraries,
+provided that they expose an API to get and set the limit on the number of threads.
+For that, one must implement a controller for this library and register it to
+`threadpoolctl`.
+
+A custom controller must be a subclass of the `LibController` class and implement
+the attributes and methods described in the docstring of `LibController`. Then this
+new controller class must be registered using the `threadpoolctl.register` function.
+An complete example can be found [here](
+  https://github.com/joblib/threadpoolctl/blob/master/tests/_pyMylib/__init__.py).
+
+### Sequential BLAS within OpenMP parallel region
+
+When one wants to have sequential BLAS calls within an OpenMP parallel region, it's
+safer to set `limits="sequential_blas_under_openmp"` since setting `limits=1` and
+`user_api="blas"` might not lead to the expected behavior in some configurations
+(e.g. OpenBLAS with the OpenMP threading layer
+https://github.com/xianyi/OpenBLAS/issues/2985).
+
+### Known Limitations
+
+- `threadpool_limits` can fail to limit the number of inner threads when nesting
+  parallel loops managed by distinct OpenMP runtime implementations (for instance
+  libgomp from GCC and libomp from clang/llvm or libiomp from ICC).
+
+  See the `test_openmp_nesting` function in [tests/test_threadpoolctl.py](
+  https://github.com/joblib/threadpoolctl/blob/master/tests/test_threadpoolctl.py)
+  for an example. More information can be found at:
+  https://github.com/jeremiedbb/Nested_OpenMP
+
+  Note however that this problem does not happen when `threadpool_limits` is
+  used to limit the number of threads used internally by BLAS calls that are
+  themselves nested under OpenMP parallel loops. `threadpool_limits` works as
+  expected, even if the inner BLAS implementation relies on a distinct OpenMP
+  implementation.
+
+- Using Intel OpenMP (ICC) and LLVM OpenMP (clang) in the same Python program
+  under Linux is known to cause problems. See the following guide for more details
+  and workarounds:
+  https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md
+
+- Setting the maximum number of threads of the OpenMP and BLAS libraries has a global
+  effect and impacts the whole Python process. There is no thread level isolation as
+  these libraries do not offer thread-local APIs to configure the number of threads to
+  use in nested parallel calls.
+
+
+## Maintainers
+
+To make a release:
+
+- Bump the version number (`__version__`) in `threadpoolctl.py` and update the
+  release date in `CHANGES.md`.
+
+- Build the distribution archives:
+
+```bash
+pip install flit
+flit build
+```
+
+and check the contents of `dist/`.
+
+- If everything is fine, make a commit for the release, tag it and push the
+tag to github:
+
+```bash
+git tag -a X.Y.Z
+git push git@github.com:joblib/threadpoolctl.git X.Y.Z
+```
+
+- Upload the wheels and source distribution to PyPI using flit. Since PyPI doesn't
+  allow password authentication anymore, the username needs to be changed to the
+  generic name `__token__`:
+
+```bash
+FLIT_USERNAME=__token__ flit publish
+```
+
+  and a PyPI token has to be passed in place of the password.
+
+- Create a PR for the release on the [conda-forge feedstock](https://github.com/conda-forge/threadpoolctl-feedstock) (or wait for the bot to make it).
+
+- Publish the release on github.
+
+### Credits
+
+The initial dynamic library introspection code was written by @anton-malakhov
+for the smp package available at https://github.com/IntelPython/smp .
+
+threadpoolctl extends this for other operating systems. Contrary to smp,
+threadpoolctl does not attempt to limit the size of Python multiprocessing
+pools (threads or processes) or set operating system-level CPU affinity
+constraints: threadpoolctl only interacts with native libraries via their
+public runtime APIs.
+
diff --git a/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/RECORD b/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/RECORD
new file mode 100644
index 0000000000000000000000000000000000000000..f8697dae6e04d8c1c68479174dbce9e117abe5f7
--- /dev/null
+++ b/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/RECORD
@@ -0,0 +1,7 @@
+__pycache__/threadpoolctl.cpython-310.pyc,,
+threadpoolctl-3.6.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+threadpoolctl-3.6.0.dist-info/METADATA,sha256=pF340H6hiD13IYOlAdfVJgdqpw38_dsnaiy9wE3vU0E,13843
+threadpoolctl-3.6.0.dist-info/RECORD,,
+threadpoolctl-3.6.0.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
+threadpoolctl-3.6.0.dist-info/licenses/LICENSE,sha256=gaxhkHUkiwblNmC2UtEOSF9GdfXQrg-X6iI3DaH34js,1507
+threadpoolctl.py,sha256=EvuVJranTS5oa37BSNwWXDWHmZsU-oaYSqGA4QgCQAs,50722
diff --git a/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/WHEEL b/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/WHEEL
new file mode 100644
index 0000000000000000000000000000000000000000..23d2d7e9a5d381ef8a375db09f82052144d1fd96
--- /dev/null
+++ b/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/WHEEL
@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: flit 3.11.0
+Root-Is-Purelib: true
+Tag: py3-none-any
diff --git a/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/licenses/LICENSE b/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/licenses/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..f2927f5f8147f137783bb5072794999e04655cfd
--- /dev/null
+++ b/lib/python3.1/site-packages/threadpoolctl-3.6.0.dist-info/licenses/LICENSE
@@ -0,0 +1,24 @@
+Copyright (c) 2019, threadpoolctl contributors
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of copyright holder nor the names of its contributors
+      may be used to endorse or promote products derived from this software
+      without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/lib/python3.1/site-packages/threadpoolctl.py b/lib/python3.1/site-packages/threadpoolctl.py
new file mode 100644
index 0000000000000000000000000000000000000000..aea33509dc5512c0e845d387e01577c22f027676
--- /dev/null
+++ b/lib/python3.1/site-packages/threadpoolctl.py
@@ -0,0 +1,1292 @@
+"""threadpoolctl
+
+This module provides utilities to introspect native libraries that relies on
+thread pools (notably BLAS and OpenMP implementations) and dynamically set the
+maximal number of threads they can use.
+"""
+
+# License: BSD 3-Clause
+
+# The code to introspect dynamically loaded libraries on POSIX systems is
+# adapted from code by Intel developer @anton-malakhov available at
+# https://github.com/IntelPython/smp (Copyright (c) 2017, Intel Corporation)
+# and also published under the BSD 3-Clause license
+import os
+import re
+import sys
+import ctypes
+import itertools
+import textwrap
+from typing import final
+import warnings
+from ctypes.util import find_library
+from abc import ABC, abstractmethod
+from functools import lru_cache
+from contextlib import ContextDecorator
+
+__version__ = "3.6.0"
+__all__ = [
+    "threadpool_limits",
+    "threadpool_info",
+    "ThreadpoolController",
+    "LibController",
+    "register",
+]
+
+
+# One can get runtime errors or even segfaults due to multiple OpenMP libraries
+# loaded simultaneously which can happen easily in Python when importing and
+# using compiled extensions built with different compilers and therefore
+# different OpenMP runtimes in the same program. In particular libiomp (used by
+# Intel ICC) and libomp used by clang/llvm tend to crash. This can happen for
+# instance when calling BLAS inside a prange. Setting the following environment
+# variable allows multiple OpenMP libraries to be loaded. It should not degrade
+# performances since we manually take care of potential over-subscription
+# performance issues, in sections of the code where nested OpenMP loops can
+# happen, by dynamically reconfiguring the inner OpenMP runtime to temporarily
+# disable it while under the scope of the outer OpenMP parallel section.
+os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "True")
+
+# Structure to cast the info on dynamically loaded library. See
+# https://linux.die.net/man/3/dl_iterate_phdr for more details.
+_SYSTEM_UINT = ctypes.c_uint64 if sys.maxsize > 2**32 else ctypes.c_uint32
+_SYSTEM_UINT_HALF = ctypes.c_uint32 if sys.maxsize > 2**32 else ctypes.c_uint16
+
+
+class _dl_phdr_info(ctypes.Structure):
+    _fields_ = [
+        ("dlpi_addr", _SYSTEM_UINT),  # Base address of object
+        ("dlpi_name", ctypes.c_char_p),  # path to the library
+        ("dlpi_phdr", ctypes.c_void_p),  # pointer on dlpi_headers
+        ("dlpi_phnum", _SYSTEM_UINT_HALF),  # number of elements in dlpi_phdr
+    ]
+
+
+# The RTLD_NOLOAD flag for loading shared libraries is not defined on Windows.
+try:
+    _RTLD_NOLOAD = os.RTLD_NOLOAD
+except AttributeError:
+    _RTLD_NOLOAD = ctypes.DEFAULT_MODE
+
+
+class LibController(ABC):
+    """Abstract base class for the individual library controllers
+
+    A library controller must expose the following class attributes:
+        - user_api : str
+            Usually the name of the library or generic specification the library
+            implements, e.g. "blas" is a specification with different implementations.
+        - internal_api : str
+            Usually the name of the library or concrete implementation of some
+            specification, e.g. "openblas" is an implementation of the "blas"
+            specification.
+        - filename_prefixes : tuple
+            Possible prefixes of the shared library's filename that allow to
+            identify the library. e.g. "libopenblas" for libopenblas.so.
+
+    and implement the following methods: `get_num_threads`, `set_num_threads` and
+    `get_version`.
+
+    Threadpoolctl loops through all the loaded shared libraries and tries to match
+    the filename of each library with the `filename_prefixes`. If a match is found, a
+    controller is instantiated and a handler to the library is stored in the `dynlib`
+    attribute as a `ctypes.CDLL` object. It can be used to access the necessary symbols
+    of the shared library to implement the above methods.
+
+    The following information will be exposed in the info dictionary:
+      - user_api : standardized API, if any, or a copy of internal_api.
+      - internal_api : implementation-specific API.
+      - num_threads : the current thread limit.
+      - prefix : prefix of the shared library's filename.
+      - filepath : path to the loaded shared library.
+      - version : version of the library (if available).
+
+    In addition, each library controller may expose internal API specific entries. They
+    must be set as attributes in the `set_additional_attributes` method.
+    """
+
+    @final
+    def __init__(self, *, filepath=None, prefix=None, parent=None):
+        """This is not meant to be overriden by subclasses."""
+        self.parent = parent
+        self.prefix = prefix
+        self.filepath = filepath
+        self.dynlib = ctypes.CDLL(filepath, mode=_RTLD_NOLOAD)
+        self._symbol_prefix, self._symbol_suffix = self._find_affixes()
+        self.version = self.get_version()
+        self.set_additional_attributes()
+
+    def info(self):
+        """Return relevant info wrapped in a dict"""
+        hidden_attrs = ("dynlib", "parent", "_symbol_prefix", "_symbol_suffix")
+        return {
+            "user_api": self.user_api,
+            "internal_api": self.internal_api,
+            "num_threads": self.num_threads,
+            **{k: v for k, v in vars(self).items() if k not in hidden_attrs},
+        }
+
+    def set_additional_attributes(self):
+        """Set additional attributes meant to be exposed in the info dict"""
+
+    @property
+    def num_threads(self):
+        """Exposes the current thread limit as a dynamic property
+
+        This is not meant to be used or overriden by subclasses.
+        """
+        return self.get_num_threads()
+
+    @abstractmethod
+    def get_num_threads(self):
+        """Return the maximum number of threads available to use"""
+
+    @abstractmethod
+    def set_num_threads(self, num_threads):
+        """Set the maximum number of threads to use"""
+
+    @abstractmethod
+    def get_version(self):
+        """Return the version of the shared library"""
+
+    def _find_affixes(self):
+        """Return the affixes for the symbols of the shared library"""
+        return "", ""
+
+    def _get_symbol(self, name):
+        """Return the symbol of the shared library accounding for the affixes"""
+        return getattr(
+            self.dynlib, f"{self._symbol_prefix}{name}{self._symbol_suffix}", None
+        )
+
+
+class OpenBLASController(LibController):
+    """Controller class for OpenBLAS"""
+
+    user_api = "blas"
+    internal_api = "openblas"
+    filename_prefixes = ("libopenblas", "libblas", "libscipy_openblas")
+
+    _symbol_prefixes = ("", "scipy_")
+    _symbol_suffixes = ("", "64_", "_64")
+
+    # All variations of "openblas_get_num_threads", accounting for the affixes
+    check_symbols = tuple(
+        f"{prefix}openblas_get_num_threads{suffix}"
+        for prefix, suffix in itertools.product(_symbol_prefixes, _symbol_suffixes)
+    )
+
+    def _find_affixes(self):
+        for prefix, suffix in itertools.product(
+            self._symbol_prefixes, self._symbol_suffixes
+        ):
+            if hasattr(self.dynlib, f"{prefix}openblas_get_num_threads{suffix}"):
+                return prefix, suffix
+
+    def set_additional_attributes(self):
+        self.threading_layer = self._get_threading_layer()
+        self.architecture = self._get_architecture()
+
+    def get_num_threads(self):
+        get_num_threads_func = self._get_symbol("openblas_get_num_threads")
+        if get_num_threads_func is not None:
+            return get_num_threads_func()
+        return None
+
+    def set_num_threads(self, num_threads):
+        set_num_threads_func = self._get_symbol("openblas_set_num_threads")
+        if set_num_threads_func is not None:
+            return set_num_threads_func(num_threads)
+        return None
+
+    def get_version(self):
+        # None means OpenBLAS is not loaded or version < 0.3.4, since OpenBLAS
+        # did not expose its version before that.
+        get_version_func = self._get_symbol("openblas_get_config")
+        if get_version_func is not None:
+            get_version_func.restype = ctypes.c_char_p
+            config = get_version_func().split()
+            if config[0] == b"OpenBLAS":
+                return config[1].decode("utf-8")
+            return None
+        return None
+
+    def _get_threading_layer(self):
+        """Return the threading layer of OpenBLAS"""
+        get_threading_layer_func = self._get_symbol("openblas_get_parallel")
+        if get_threading_layer_func is not None:
+            threading_layer = get_threading_layer_func()
+            if threading_layer == 2:
+                return "openmp"
+            elif threading_layer == 1:
+                return "pthreads"
+            return "disabled"
+        return "unknown"
+
+    def _get_architecture(self):
+        """Return the architecture detected by OpenBLAS"""
+        get_architecture_func = self._get_symbol("openblas_get_corename")
+        if get_architecture_func is not None:
+            get_architecture_func.restype = ctypes.c_char_p
+            return get_architecture_func().decode("utf-8")
+        return None
+
+
+class BLISController(LibController):
+    """Controller class for BLIS"""
+
+    user_api = "blas"
+    internal_api = "blis"
+    filename_prefixes = ("libblis", "libblas")
+    check_symbols = (
+        "bli_thread_get_num_threads",
+        "bli_thread_set_num_threads",
+        "bli_info_get_version_str",
+        "bli_info_get_enable_openmp",
+        "bli_info_get_enable_pthreads",
+        "bli_arch_query_id",
+        "bli_arch_string",
+    )
+
+    def set_additional_attributes(self):
+        self.threading_layer = self._get_threading_layer()
+        self.architecture = self._get_architecture()
+
+    def get_num_threads(self):
+        get_func = getattr(self.dynlib, "bli_thread_get_num_threads", lambda: None)
+        num_threads = get_func()
+        # by default BLIS is single-threaded and get_num_threads
+        # returns -1. We map it to 1 for consistency with other libraries.
+        return 1 if num_threads == -1 else num_threads
+
+    def set_num_threads(self, num_threads):
+        set_func = getattr(
+            self.dynlib, "bli_thread_set_num_threads", lambda num_threads: None
+        )
+        return set_func(num_threads)
+
+    def get_version(self):
+        get_version_ = getattr(self.dynlib, "bli_info_get_version_str", None)
+        if get_version_ is None:
+            return None
+
+        get_version_.restype = ctypes.c_char_p
+        return get_version_().decode("utf-8")
+
+    def _get_threading_layer(self):
+        """Return the threading layer of BLIS"""
+        if getattr(self.dynlib, "bli_info_get_enable_openmp", lambda: False)():
+            return "openmp"
+        elif getattr(self.dynlib, "bli_info_get_enable_pthreads", lambda: False)():
+            return "pthreads"
+        return "disabled"
+
+    def _get_architecture(self):
+        """Return the architecture detected by BLIS"""
+        bli_arch_query_id = getattr(self.dynlib, "bli_arch_query_id", None)
+        bli_arch_string = getattr(self.dynlib, "bli_arch_string", None)
+        if bli_arch_query_id is None or bli_arch_string is None:
+            return None
+
+        # the true restype should be BLIS' arch_t (enum) but int should work
+        # for us:
+        bli_arch_query_id.restype = ctypes.c_int
+        bli_arch_string.restype = ctypes.c_char_p
+        return bli_arch_string(bli_arch_query_id()).decode("utf-8")
+
+
+class FlexiBLASController(LibController):
+    """Controller class for FlexiBLAS"""
+
+    user_api = "blas"
+    internal_api = "flexiblas"
+    filename_prefixes = ("libflexiblas",)
+    check_symbols = (
+        "flexiblas_get_num_threads",
+        "flexiblas_set_num_threads",
+        "flexiblas_get_version",
+        "flexiblas_list",
+        "flexiblas_list_loaded",
+        "flexiblas_current_backend",
+    )
+
+    @property
+    def loaded_backends(self):
+        return self._get_backend_list(loaded=True)
+
+    @property
+    def current_backend(self):
+        return self._get_current_backend()
+
+    def info(self):
+        """Return relevant info wrapped in a dict"""
+        # We override the info method because the loaded and current backends
+        # are dynamic properties
+        exposed_attrs = super().info()
+        exposed_attrs["loaded_backends"] = self.loaded_backends
+        exposed_attrs["current_backend"] = self.current_backend
+
+        return exposed_attrs
+
+    def set_additional_attributes(self):
+        self.available_backends = self._get_backend_list(loaded=False)
+
+    def get_num_threads(self):
+        get_func = getattr(self.dynlib, "flexiblas_get_num_threads", lambda: None)
+        num_threads = get_func()
+        # by default BLIS is single-threaded and get_num_threads
+        # returns -1. We map it to 1 for consistency with other libraries.
+        return 1 if num_threads == -1 else num_threads
+
+    def set_num_threads(self, num_threads):
+        set_func = getattr(
+            self.dynlib, "flexiblas_set_num_threads", lambda num_threads: None
+        )
+        return set_func(num_threads)
+
+    def get_version(self):
+        get_version_ = getattr(self.dynlib, "flexiblas_get_version", None)
+        if get_version_ is None:
+            return None
+
+        major = ctypes.c_int()
+        minor = ctypes.c_int()
+        patch = ctypes.c_int()
+        get_version_(ctypes.byref(major), ctypes.byref(minor), ctypes.byref(patch))
+        return f"{major.value}.{minor.value}.{patch.value}"
+
+    def _get_backend_list(self, loaded=False):
+        """Return the list of available backends for FlexiBLAS.
+
+        If loaded is False, return the list of available backends from the FlexiBLAS
+        configuration. If loaded is True, return the list of actually loaded backends.
+        """
+        func_name = f"flexiblas_list{'_loaded' if loaded else ''}"
+        get_backend_list_ = getattr(self.dynlib, func_name, None)
+        if get_backend_list_ is None:
+            return None
+
+        n_backends = get_backend_list_(None, 0, 0)
+
+        backends = []
+        for i in range(n_backends):
+            backend_name = ctypes.create_string_buffer(1024)
+            get_backend_list_(backend_name, 1024, i)
+            if backend_name.value.decode("utf-8") != "__FALLBACK__":
+                # We don't know when to expect __FALLBACK__ but it is not a real
+                # backend and does not show up when running flexiblas list.
+                backends.append(backend_name.value.decode("utf-8"))
+        return backends
+
+    def _get_current_backend(self):
+        """Return the backend of FlexiBLAS"""
+        get_backend_ = getattr(self.dynlib, "flexiblas_current_backend", None)
+        if get_backend_ is None:
+            return None
+
+        backend = ctypes.create_string_buffer(1024)
+        get_backend_(backend, ctypes.sizeof(backend))
+        return backend.value.decode("utf-8")
+
+    def switch_backend(self, backend):
+        """Switch the backend of FlexiBLAS
+
+        Parameters
+        ----------
+        backend : str
+            The name or the path to the shared library of the backend to switch to. If
+            the backend is not already loaded, it will be loaded first.
+        """
+        if backend not in self.loaded_backends:
+            if backend in self.available_backends:
+                load_func = getattr(self.dynlib, "flexiblas_load_backend", lambda _: -1)
+            else:  # assume backend is a path to a shared library
+                load_func = getattr(
+                    self.dynlib, "flexiblas_load_backend_library", lambda _: -1
+                )
+            res = load_func(str(backend).encode("utf-8"))
+            if res == -1:
+                raise RuntimeError(
+                    f"Failed to load backend {backend!r}. It must either be the name of"
+                    " a backend available in the FlexiBLAS configuration "
+                    f"{self.available_backends} or the path to a valid shared library."
+                )
+
+            # Trigger a new search of loaded shared libraries since loading a new
+            # backend caused a dlopen.
+            self.parent._load_libraries()
+
+        switch_func = getattr(self.dynlib, "flexiblas_switch", lambda _: -1)
+        idx = self.loaded_backends.index(backend)
+        res = switch_func(idx)
+        if res == -1:
+            raise RuntimeError(f"Failed to switch to backend {backend!r}.")
+
+
+class MKLController(LibController):
+    """Controller class for MKL"""
+
+    user_api = "blas"
+    internal_api = "mkl"
+    filename_prefixes = ("libmkl_rt", "mkl_rt", "libblas")
+    check_symbols = (
+        "MKL_Get_Max_Threads",
+        "MKL_Set_Num_Threads",
+        "MKL_Get_Version_String",
+        "MKL_Set_Threading_Layer",
+    )
+
+    def set_additional_attributes(self):
+        self.threading_layer = self._get_threading_layer()
+
+    def get_num_threads(self):
+        get_func = getattr(self.dynlib, "MKL_Get_Max_Threads", lambda: None)
+        return get_func()
+
+    def set_num_threads(self, num_threads):
+        set_func = getattr(self.dynlib, "MKL_Set_Num_Threads", lambda num_threads: None)
+        return set_func(num_threads)
+
+    def get_version(self):
+        if not hasattr(self.dynlib, "MKL_Get_Version_String"):
+            return None
+
+        res = ctypes.create_string_buffer(200)
+        self.dynlib.MKL_Get_Version_String(res, 200)
+
+        version = res.value.decode("utf-8")
+        group = re.search(r"Version ([^ ]+) ", version)
+        if group is not None:
+            version = group.groups()[0]
+        return version.strip()
+
+    def _get_threading_layer(self):
+        """Return the threading layer of MKL"""
+        # The function mkl_set_threading_layer returns the current threading
+        # layer. Calling it with an invalid threading layer allows us to safely
+        # get the threading layer
+        set_threading_layer = getattr(
+            self.dynlib, "MKL_Set_Threading_Layer", lambda layer: -1
+        )
+        layer_map = {
+            0: "intel",
+            1: "sequential",
+            2: "pgi",
+            3: "gnu",
+            4: "tbb",
+            -1: "not specified",
+        }
+        return layer_map[set_threading_layer(-1)]
+
+
+class OpenMPController(LibController):
+    """Controller class for OpenMP"""
+
+    user_api = "openmp"
+    internal_api = "openmp"
+    filename_prefixes = ("libiomp", "libgomp", "libomp", "vcomp")
+    check_symbols = (
+        "omp_get_max_threads",
+        "omp_get_num_threads",
+    )
+
+    def get_num_threads(self):
+        get_func = getattr(self.dynlib, "omp_get_max_threads", lambda: None)
+        return get_func()
+
+    def set_num_threads(self, num_threads):
+        set_func = getattr(self.dynlib, "omp_set_num_threads", lambda num_threads: None)
+        return set_func(num_threads)
+
+    def get_version(self):
+        # There is no way to get the version number programmatically in OpenMP.
+        return None
+
+
+# Controllers for the libraries that we'll look for in the loaded libraries.
+# Third party libraries can register their own controllers.
+_ALL_CONTROLLERS = [
+    OpenBLASController,
+    BLISController,
+    MKLController,
+    OpenMPController,
+    FlexiBLASController,
+]
+
+# Helpers for the doc and test names
+_ALL_USER_APIS = list(set(lib.user_api for lib in _ALL_CONTROLLERS))
+_ALL_INTERNAL_APIS = [lib.internal_api for lib in _ALL_CONTROLLERS]
+_ALL_PREFIXES = list(
+    set(prefix for lib in _ALL_CONTROLLERS for prefix in lib.filename_prefixes)
+)
+_ALL_BLAS_LIBRARIES = [
+    lib.internal_api for lib in _ALL_CONTROLLERS if lib.user_api == "blas"
+]
+_ALL_OPENMP_LIBRARIES = OpenMPController.filename_prefixes
+
+
+def register(controller):
+    """Register a new controller"""
+    _ALL_CONTROLLERS.append(controller)
+    _ALL_USER_APIS.append(controller.user_api)
+    _ALL_INTERNAL_APIS.append(controller.internal_api)
+    _ALL_PREFIXES.extend(controller.filename_prefixes)
+
+
+def _format_docstring(*args, **kwargs):
+    def decorator(o):
+        if o.__doc__ is not None:
+            o.__doc__ = o.__doc__.format(*args, **kwargs)
+        return o
+
+    return decorator
+
+
+@lru_cache(maxsize=10000)
+def _realpath(filepath):
+    """Small caching wrapper around os.path.realpath to limit system calls"""
+    return os.path.realpath(filepath)
+
+
+@_format_docstring(USER_APIS=list(_ALL_USER_APIS), INTERNAL_APIS=_ALL_INTERNAL_APIS)
+def threadpool_info():
+    """Return the maximal number of threads for each detected library.
+
+    Return a list with all the supported libraries that have been found. Each
+    library is represented by a dict with the following information:
+
+      - "user_api" : user API. Possible values are {USER_APIS}.
+      - "internal_api": internal API. Possible values are {INTERNAL_APIS}.
+      - "prefix" : filename prefix of the specific implementation.
+      - "filepath": path to the loaded library.
+      - "version": version of the library (if available).
+      - "num_threads": the current thread limit.
+
+    In addition, each library may contain internal_api specific entries.
+    """
+    return ThreadpoolController().info()
+
+
+class _ThreadpoolLimiter:
+    """The guts of ThreadpoolController.limit
+
+    Refer to the docstring of ThreadpoolController.limit for more details.
+
+    It will only act on the library controllers held by the provided `controller`.
+    Using the default constructor sets the limits right away such that it can be used as
+    a callable. Setting the limits can be delayed by using the `wrap` class method such
+    that it can be used as a decorator.
+    """
+
+    def __init__(self, controller, *, limits=None, user_api=None):
+        self._controller = controller
+        self._limits, self._user_api, self._prefixes = self._check_params(
+            limits, user_api
+        )
+        self._original_info = self._controller.info()
+        self._set_threadpool_limits()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.restore_original_limits()
+
+    @classmethod
+    def wrap(cls, controller, *, limits=None, user_api=None):
+        """Return an instance of this class that can be used as a decorator"""
+        return _ThreadpoolLimiterDecorator(
+            controller=controller, limits=limits, user_api=user_api
+        )
+
+    def restore_original_limits(self):
+        """Set the limits back to their original values"""
+        for lib_controller, original_info in zip(
+            self._controller.lib_controllers, self._original_info
+        ):
+            lib_controller.set_num_threads(original_info["num_threads"])
+
+    # Alias of `restore_original_limits` for backward compatibility
+    unregister = restore_original_limits
+
+    def get_original_num_threads(self):
+        """Original num_threads from before calling threadpool_limits
+
+        Return a dict `{user_api: num_threads}`.
+        """
+        num_threads = {}
+        warning_apis = []
+
+        for user_api in self._user_api:
+            limits = [
+                lib_info["num_threads"]
+                for lib_info in self._original_info
+                if lib_info["user_api"] == user_api
+            ]
+            limits = set(limits)
+            n_limits = len(limits)
+
+            if n_limits == 1:
+                limit = limits.pop()
+            elif n_limits == 0:
+                limit = None
+            else:
+                limit = min(limits)
+                warning_apis.append(user_api)
+
+            num_threads[user_api] = limit
+
+        if warning_apis:
+            warnings.warn(
+                "Multiple value possible for following user apis: "
+                + ", ".join(warning_apis)
+                + ". Returning the minimum."
+            )
+
+        return num_threads
+
+    def _check_params(self, limits, user_api):
+        """Suitable values for the _limits, _user_api and _prefixes attributes"""
+
+        if isinstance(limits, str) and limits == "sequential_blas_under_openmp":
+            (
+                limits,
+                user_api,
+            ) = self._controller._get_params_for_sequential_blas_under_openmp().values()
+
+        if limits is None or isinstance(limits, int):
+            if user_api is None:
+                user_api = _ALL_USER_APIS
+            elif user_api in _ALL_USER_APIS:
+                user_api = [user_api]
+            else:
+                raise ValueError(
+                    f"user_api must be either in {_ALL_USER_APIS} or None. Got "
+                    f"{user_api} instead."
+                )
+
+            if limits is not None:
+                limits = {api: limits for api in user_api}
+            prefixes = []
+        else:
+            if isinstance(limits, list):
+                # This should be a list of dicts of library info, for
+                # compatibility with the result from threadpool_info.
+                limits = {
+                    lib_info["prefix"]: lib_info["num_threads"] for lib_info in limits
+                }
+            elif isinstance(limits, ThreadpoolController):
+                # To set the limits from the library controllers of a
+                # ThreadpoolController object.
+                limits = {
+                    lib_controller.prefix: lib_controller.num_threads
+                    for lib_controller in limits.lib_controllers
+                }
+
+            if not isinstance(limits, dict):
+                raise TypeError(
+                    "limits must either be an int, a list, a dict, or "
+                    f"'sequential_blas_under_openmp'. Got {type(limits)} instead"
+                )
+
+            # With a dictionary, can set both specific limit for given
+            # libraries and global limit for user_api. Fetch each separately.
+            prefixes = [prefix for prefix in limits if prefix in _ALL_PREFIXES]
+            user_api = [api for api in limits if api in _ALL_USER_APIS]
+
+        return limits, user_api, prefixes
+
+    def _set_threadpool_limits(self):
+        """Change the maximal number of threads in selected thread pools.
+
+        Return a list with all the supported libraries that have been found
+        matching `self._prefixes` and `self._user_api`.
+        """
+        if self._limits is None:
+            return
+
+        for lib_controller in self._controller.lib_controllers:
+            # self._limits is a dict {key: num_threads} where key is either
+            # a prefix or a user_api. If a library matches both, the limit
+            # corresponding to the prefix is chosen.
+            if lib_controller.prefix in self._limits:
+                num_threads = self._limits[lib_controller.prefix]
+            elif lib_controller.user_api in self._limits:
+                num_threads = self._limits[lib_controller.user_api]
+            else:
+                continue
+
+            if num_threads is not None:
+                lib_controller.set_num_threads(num_threads)
+
+
+class _ThreadpoolLimiterDecorator(_ThreadpoolLimiter, ContextDecorator):
+    """Same as _ThreadpoolLimiter but to be used as a decorator"""
+
+    def __init__(self, controller, *, limits=None, user_api=None):
+        self._limits, self._user_api, self._prefixes = self._check_params(
+            limits, user_api
+        )
+        self._controller = controller
+
+    def __enter__(self):
+        # we need to set the limits here and not in the __init__ because we want the
+        # limits to be set when calling the decorated function, not when creating the
+        # decorator.
+        self._original_info = self._controller.info()
+        self._set_threadpool_limits()
+        return self
+
+
+@_format_docstring(
+    USER_APIS=", ".join(f'"{api}"' for api in _ALL_USER_APIS),
+    BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES),
+    OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES),
+)
+class threadpool_limits(_ThreadpoolLimiter):
+    """Change the maximal number of threads that can be used in thread pools.
+
+    This object can be used either as a callable (the construction of this object
+    limits the number of threads), as a context manager in a `with` block to
+    automatically restore the original state of the controlled libraries when exiting
+    the block, or as a decorator through its `wrap` method.
+
+    Set the maximal number of threads that can be used in thread pools used in
+    the supported libraries to `limit`. This function works for libraries that
+    are already loaded in the interpreter and can be changed dynamically.
+
+    This effect is global and impacts the whole Python process. There is no thread level
+    isolation as these libraries do not offer thread-local APIs to configure the number
+    of threads to use in nested parallel calls.
+
+    Parameters
+    ----------
+    limits : int, dict, 'sequential_blas_under_openmp' or None (default=None)
+        The maximal number of threads that can be used in thread pools
+
+        - If int, sets the maximum number of threads to `limits` for each
+          library selected by `user_api`.
+
+        - If it is a dictionary `{{key: max_threads}}`, this function sets a
+          custom maximum number of threads for each `key` which can be either a
+          `user_api` or a `prefix` for a specific library.
+
+        - If 'sequential_blas_under_openmp', it will chose the appropriate `limits`
+          and `user_api` parameters for the specific use case of sequential BLAS
+          calls within an OpenMP parallel region. The `user_api` parameter is
+          ignored.
+
+        - If None, this function does not do anything.
+
+    user_api : {USER_APIS} or None (default=None)
+        APIs of libraries to limit. Used only if `limits` is an int.
+
+        - If "blas", it will only limit BLAS supported libraries ({BLAS_LIBS}).
+
+        - If "openmp", it will only limit OpenMP supported libraries
+          ({OPENMP_LIBS}). Note that it can affect the number of threads used
+          by the BLAS libraries if they rely on OpenMP.
+
+        - If None, this function will apply to all supported libraries.
+    """
+
+    def __init__(self, limits=None, user_api=None):
+        super().__init__(ThreadpoolController(), limits=limits, user_api=user_api)
+
+    @classmethod
+    def wrap(cls, limits=None, user_api=None):
+        return super().wrap(ThreadpoolController(), limits=limits, user_api=user_api)
+
+
+class ThreadpoolController:
+    """Collection of LibController objects for all loaded supported libraries
+
+    Attributes
+    ----------
+    lib_controllers : list of `LibController` objects
+        The list of library controllers of all loaded supported libraries.
+    """
+
+    # Cache for libc under POSIX and a few system libraries under Windows.
+    # We use a class level cache instead of an instance level cache because
+    # it's very unlikely that a shared library will be unloaded and reloaded
+    # during the lifetime of a program.
+    _system_libraries = dict()
+
+    def __init__(self):
+        self.lib_controllers = []
+        self._load_libraries()
+        self._warn_if_incompatible_openmp()
+
+    @classmethod
+    def _from_controllers(cls, lib_controllers):
+        new_controller = cls.__new__(cls)
+        new_controller.lib_controllers = lib_controllers
+        return new_controller
+
+    def info(self):
+        """Return lib_controllers info as a list of dicts"""
+        return [lib_controller.info() for lib_controller in self.lib_controllers]
+
+    def select(self, **kwargs):
+        """Return a ThreadpoolController containing a subset of its current
+        library controllers
+
+        It will select all libraries matching at least one pair (key, value) from kwargs
+        where key is an entry of the library info dict (like "user_api", "internal_api",
+        "prefix", ...) and value is the value or a list of acceptable values for that
+        entry.
+
+        For instance, `ThreadpoolController().select(internal_api=["blis", "openblas"])`
+        will select all library controllers whose internal_api is either "blis" or
+        "openblas".
+        """
+        for key, vals in kwargs.items():
+            kwargs[key] = [vals] if not isinstance(vals, list) else vals
+
+        lib_controllers = [
+            lib_controller
+            for lib_controller in self.lib_controllers
+            if any(
+                getattr(lib_controller, key, None) in vals
+                for key, vals in kwargs.items()
+            )
+        ]
+
+        return ThreadpoolController._from_controllers(lib_controllers)
+
+    def _get_params_for_sequential_blas_under_openmp(self):
+        """Return appropriate params to use for a sequential BLAS call in an OpenMP loop
+
+        This function takes into account the unexpected behavior of OpenBLAS with the
+        OpenMP threading layer.
+        """
+        if self.select(
+            internal_api="openblas", threading_layer="openmp"
+        ).lib_controllers:
+            return {"limits": None, "user_api": None}
+        return {"limits": 1, "user_api": "blas"}
+
+    @_format_docstring(
+        USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS),
+        BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES),
+        OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES),
+    )
+    def limit(self, *, limits=None, user_api=None):
+        """Change the maximal number of threads that can be used in thread pools.
+
+        This function returns an object that can be used either as a callable (the
+        construction of this object limits the number of threads) or as a context
+        manager, in a `with` block to automatically restore the original state of the
+        controlled libraries when exiting the block.
+
+        Set the maximal number of threads that can be used in thread pools used in
+        the supported libraries to `limits`. This function works for libraries that
+        are already loaded in the interpreter and can be changed dynamically.
+
+        This effect is global and impacts the whole Python process. There is no thread
+        level isolation as these libraries do not offer thread-local APIs to configure
+        the number of threads to use in nested parallel calls.
+
+        Parameters
+        ----------
+        limits : int, dict, 'sequential_blas_under_openmp' or None (default=None)
+            The maximal number of threads that can be used in thread pools
+
+            - If int, sets the maximum number of threads to `limits` for each
+              library selected by `user_api`.
+
+            - If it is a dictionary `{{key: max_threads}}`, this function sets a
+              custom maximum number of threads for each `key` which can be either a
+              `user_api` or a `prefix` for a specific library.
+
+            - If 'sequential_blas_under_openmp', it will chose the appropriate `limits`
+              and `user_api` parameters for the specific use case of sequential BLAS
+              calls within an OpenMP parallel region. The `user_api` parameter is
+              ignored.
+
+            - If None, this function does not do anything.
+
+        user_api : {USER_APIS} or None (default=None)
+            APIs of libraries to limit. Used only if `limits` is an int.
+
+            - If "blas", it will only limit BLAS supported libraries ({BLAS_LIBS}).
+
+            - If "openmp", it will only limit OpenMP supported libraries
+              ({OPENMP_LIBS}). Note that it can affect the number of threads used
+              by the BLAS libraries if they rely on OpenMP.
+
+            - If None, this function will apply to all supported libraries.
+        """
+        return _ThreadpoolLimiter(self, limits=limits, user_api=user_api)
+
+    @_format_docstring(
+        USER_APIS=", ".join('"{}"'.format(api) for api in _ALL_USER_APIS),
+        BLAS_LIBS=", ".join(_ALL_BLAS_LIBRARIES),
+        OPENMP_LIBS=", ".join(_ALL_OPENMP_LIBRARIES),
+    )
+    def wrap(self, *, limits=None, user_api=None):
+        """Change the maximal number of threads that can be used in thread pools.
+
+        This function returns an object that can be used as a decorator.
+
+        Set the maximal number of threads that can be used in thread pools used in
+        the supported libraries to `limits`. This function works for libraries that
+        are already loaded in the interpreter and can be changed dynamically.
+
+        Parameters
+        ----------
+        limits : int, dict or None (default=None)
+            The maximal number of threads that can be used in thread pools
+
+            - If int, sets the maximum number of threads to `limits` for each
+              library selected by `user_api`.
+
+            - If it is a dictionary `{{key: max_threads}}`, this function sets a
+              custom maximum number of threads for each `key` which can be either a
+              `user_api` or a `prefix` for a specific library.
+
+            - If None, this function does not do anything.
+
+        user_api : {USER_APIS} or None (default=None)
+            APIs of libraries to limit. Used only if `limits` is an int.
+
+            - If "blas", it will only limit BLAS supported libraries ({BLAS_LIBS}).
+
+            - If "openmp", it will only limit OpenMP supported libraries
+              ({OPENMP_LIBS}). Note that it can affect the number of threads used
+              by the BLAS libraries if they rely on OpenMP.
+
+            - If None, this function will apply to all supported libraries.
+        """
+        return _ThreadpoolLimiter.wrap(self, limits=limits, user_api=user_api)
+
+    def __len__(self):
+        return len(self.lib_controllers)
+
+    def _load_libraries(self):
+        """Loop through loaded shared libraries and store the supported ones"""
+        if sys.platform == "darwin":
+            self._find_libraries_with_dyld()
+        elif sys.platform == "win32":
+            self._find_libraries_with_enum_process_module_ex()
+        elif "pyodide" in sys.modules:
+            self._find_libraries_pyodide()
+        else:
+            self._find_libraries_with_dl_iterate_phdr()
+
+    def _find_libraries_with_dl_iterate_phdr(self):
+        """Loop through loaded libraries and return binders on supported ones
+
+        This function is expected to work on POSIX system only.
+        This code is adapted from code by Intel developer @anton-malakhov
+        available at https://github.com/IntelPython/smp
+
+        Copyright (c) 2017, Intel Corporation published under the BSD 3-Clause
+        license
+        """
+        libc = self._get_libc()
+        if not hasattr(libc, "dl_iterate_phdr"):  # pragma: no cover
+            warnings.warn(
+                "Could not find dl_iterate_phdr in the C standard library.",
+                RuntimeWarning,
+            )
+            return []
+
+        # Callback function for `dl_iterate_phdr` which is called for every
+        # library loaded in the current process until it returns 1.
+        def match_library_callback(info, size, data):
+            # Get the path of the current library
+            filepath = info.contents.dlpi_name
+            if filepath:
+                filepath = filepath.decode("utf-8")
+
+                # Store the library controller if it is supported and selected
+                self._make_controller_from_path(filepath)
+            return 0
+
+        c_func_signature = ctypes.CFUNCTYPE(
+            ctypes.c_int,  # Return type
+            ctypes.POINTER(_dl_phdr_info),
+            ctypes.c_size_t,
+            ctypes.c_char_p,
+        )
+        c_match_library_callback = c_func_signature(match_library_callback)
+
+        data = ctypes.c_char_p(b"")
+        libc.dl_iterate_phdr(c_match_library_callback, data)
+
+    def _find_libraries_with_dyld(self):
+        """Loop through loaded libraries and return binders on supported ones
+
+        This function is expected to work on OSX system only
+        """
+        libc = self._get_libc()
+        if not hasattr(libc, "_dyld_image_count"):  # pragma: no cover
+            warnings.warn(
+                "Could not find _dyld_image_count in the C standard library.",
+                RuntimeWarning,
+            )
+            return []
+
+        n_dyld = libc._dyld_image_count()
+        libc._dyld_get_image_name.restype = ctypes.c_char_p
+
+        for i in range(n_dyld):
+            filepath = ctypes.string_at(libc._dyld_get_image_name(i))
+            filepath = filepath.decode("utf-8")
+
+            # Store the library controller if it is supported and selected
+            self._make_controller_from_path(filepath)
+
+    def _find_libraries_with_enum_process_module_ex(self):
+        """Loop through loaded libraries and return binders on supported ones
+
+        This function is expected to work on windows system only.
+        This code is adapted from code by Philipp Hagemeister @phihag available
+        at https://stackoverflow.com/questions/17474574
+        """
+        from ctypes.wintypes import DWORD, HMODULE, MAX_PATH
+
+        PROCESS_QUERY_INFORMATION = 0x0400
+        PROCESS_VM_READ = 0x0010
+
+        LIST_LIBRARIES_ALL = 0x03
+
+        ps_api = self._get_windll("Psapi")
+        kernel_32 = self._get_windll("kernel32")
+
+        h_process = kernel_32.OpenProcess(
+            PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, False, os.getpid()
+        )
+        if not h_process:  # pragma: no cover
+            raise OSError(f"Could not open PID {os.getpid()}")
+
+        try:
+            buf_count = 256
+            needed = DWORD()
+            # Grow the buffer until it becomes large enough to hold all the
+            # module headers
+            while True:
+                buf = (HMODULE * buf_count)()
+                buf_size = ctypes.sizeof(buf)
+                if not ps_api.EnumProcessModulesEx(
+                    h_process,
+                    ctypes.byref(buf),
+                    buf_size,
+                    ctypes.byref(needed),
+                    LIST_LIBRARIES_ALL,
+                ):
+                    raise OSError("EnumProcessModulesEx failed")
+                if buf_size >= needed.value:
+                    break
+                buf_count = needed.value // (buf_size // buf_count)
+
+            count = needed.value // (buf_size // buf_count)
+            h_modules = map(HMODULE, buf[:count])
+
+            # Loop through all the module headers and get the library path
+            # Allocate a buffer for the path 10 times the size of MAX_PATH to take
+            # into account long path names.
+            max_path = 10 * MAX_PATH
+            buf = ctypes.create_unicode_buffer(max_path)
+            n_size = DWORD()
+            for h_module in h_modules:
+                # Get the path of the current module
+                if not ps_api.GetModuleFileNameExW(
+                    h_process, h_module, ctypes.byref(buf), ctypes.byref(n_size)
+                ):
+                    raise OSError("GetModuleFileNameEx failed")
+                filepath = buf.value
+
+                if len(filepath) == max_path:  # pragma: no cover
+                    warnings.warn(
+                        "Could not get the full path of a dynamic library (path too "
+                        "long). This library will be ignored and threadpoolctl might "
+                        "not be able to control or display information about all "
+                        f"loaded libraries. Here's the truncated path: {filepath!r}",
+                        RuntimeWarning,
+                    )
+                else:
+                    # Store the library controller if it is supported and selected
+                    self._make_controller_from_path(filepath)
+        finally:
+            kernel_32.CloseHandle(h_process)
+
+    def _find_libraries_pyodide(self):
+        """Pyodide specific implementation for finding loaded libraries.
+
+        Adapted from suggestion in https://github.com/joblib/threadpoolctl/pull/169#issuecomment-1946696449.
+
+        One day, we may have a simpler solution. libc dl_iterate_phdr needs to
+        be implemented in Emscripten and exposed in Pyodide, see
+        https://github.com/emscripten-core/emscripten/issues/21354 for more
+        details.
+        """
+        try:
+            from pyodide_js._module import LDSO
+        except ImportError:
+            warnings.warn(
+                "Unable to import LDSO from pyodide_js._module. This should never "
+                "happen."
+            )
+            return
+
+        for filepath in LDSO.loadedLibsByName.as_object_map():
+            # Some libraries are duplicated by Pyodide and do not exist in the
+            # filesystem, so we first check for the existence of the file. For
+            # more details, see
+            # https://github.com/joblib/threadpoolctl/pull/169#issuecomment-1947946728
+            if os.path.exists(filepath):
+                self._make_controller_from_path(filepath)
+
+    def _make_controller_from_path(self, filepath):
+        """Store a library controller if it is supported and selected"""
+        # Required to resolve symlinks
+        filepath = _realpath(filepath)
+        # `lower` required to take account of OpenMP dll case on Windows
+        # (vcomp, VCOMP, Vcomp, ...)
+        filename = os.path.basename(filepath).lower()
+
+        # Loop through supported libraries to find if this filename corresponds
+        # to a supported one.
+        for controller_class in _ALL_CONTROLLERS:
+            # check if filename matches a supported prefix
+            prefix = self._check_prefix(filename, controller_class.filename_prefixes)
+
+            # filename does not match any of the prefixes of the candidate
+            # library. move to next library.
+            if prefix is None:
+                continue
+
+            # workaround for BLAS libraries packaged by conda-forge on windows, which
+            # are all renamed "libblas.dll". We thus have to check to which BLAS
+            # implementation it actually corresponds looking for implementation
+            # specific symbols.
+            if prefix == "libblas":
+                if filename.endswith(".dll"):
+                    libblas = ctypes.CDLL(filepath, _RTLD_NOLOAD)
+                    if not any(
+                        hasattr(libblas, func)
+                        for func in controller_class.check_symbols
+                    ):
+                        continue
+                else:
+                    # We ignore libblas on other platforms than windows because there
+                    # might be a libblas dso comming with openblas for instance that
+                    # can't be used to instantiate a pertinent LibController (many
+                    # symbols are missing) and would create confusion by making a
+                    # duplicate entry in threadpool_info.
+                    continue
+
+            # filename matches a prefix. Now we check if the library has the symbols we
+            # are looking for. If none of the symbols exists, it's very likely not the
+            # expected library (e.g. a library having a common prefix with one of the
+            # our supported libraries). Otherwise, create and store the library
+            # controller.
+            lib_controller = controller_class(
+                filepath=filepath, prefix=prefix, parent=self
+            )
+
+            if filepath in (lib.filepath for lib in self.lib_controllers):
+                # We already have a controller for this library.
+                continue
+
+            if not hasattr(controller_class, "check_symbols") or any(
+                hasattr(lib_controller.dynlib, func)
+                for func in controller_class.check_symbols
+            ):
+                self.lib_controllers.append(lib_controller)
+
+    def _check_prefix(self, library_basename, filename_prefixes):
+        """Return the prefix library_basename starts with
+
+        Return None if none matches.
+        """
+        for prefix in filename_prefixes:
+            if library_basename.startswith(prefix):
+                return prefix
+        return None
+
+    def _warn_if_incompatible_openmp(self):
+        """Raise a warning if llvm-OpenMP and intel-OpenMP are both loaded"""
+        prefixes = [lib_controller.prefix for lib_controller in self.lib_controllers]
+        msg = textwrap.dedent(
+            """
+            Found Intel OpenMP ('libiomp') and LLVM OpenMP ('libomp') loaded at
+            the same time. Both libraries are known to be incompatible and this
+            can cause random crashes or deadlocks on Linux when loaded in the
+            same Python program.
+            Using threadpoolctl may cause crashes or deadlocks. For more
+            information and possible workarounds, please see
+                https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md
+            """
+        )
+        if "libomp" in prefixes and "libiomp" in prefixes:
+            warnings.warn(msg, RuntimeWarning)
+
+    @classmethod
+    def _get_libc(cls):
+        """Load the lib-C for unix systems."""
+        libc = cls._system_libraries.get("libc")
+        if libc is None:
+            # Remark: If libc is statically linked or if Python is linked against an
+            # alternative implementation of libc like musl, find_library will return
+            # None and CDLL will load the main program itself which should contain the
+            # libc symbols. We still name it libc for convenience.
+            # If the main program does not contain the libc symbols, it's ok because
+            # we check their presence later anyway.
+            libc = ctypes.CDLL(find_library("c"), mode=_RTLD_NOLOAD)
+            cls._system_libraries["libc"] = libc
+        return libc
+
+    @classmethod
+    def _get_windll(cls, dll_name):
+        """Load a windows DLL"""
+        dll = cls._system_libraries.get(dll_name)
+        if dll is None:
+            dll = ctypes.WinDLL(f"{dll_name}.dll")
+            cls._system_libraries[dll_name] = dll
+        return dll
+
+
+def _main():
+    """Commandline interface to display thread-pool information and exit."""
+    import argparse
+    import importlib
+    import json
+    import sys
+
+    parser = argparse.ArgumentParser(
+        usage="python -m threadpoolctl -i numpy scipy.linalg xgboost",
+        description="Display thread-pool information and exit.",
+    )
+    parser.add_argument(
+        "-i",
+        "--import",
+        dest="modules",
+        nargs="*",
+        default=(),
+        help="Python modules to import before introspecting thread-pools.",
+    )
+    parser.add_argument(
+        "-c",
+        "--command",
+        help="a Python statement to execute before introspecting thread-pools.",
+    )
+
+    options = parser.parse_args(sys.argv[1:])
+    for module in options.modules:
+        try:
+            importlib.import_module(module, package=None)
+        except ImportError:
+            print("WARNING: could not import", module, file=sys.stderr)
+
+    if options.command:
+        exec(options.command)
+
+    print(json.dumps(threadpool_info(), indent=2))
+
+
+if __name__ == "__main__":
+    _main()
diff --git a/lib/python3.1/site-packages/typing_extensions.py b/lib/python3.1/site-packages/typing_extensions.py
new file mode 100644
index 0000000000000000000000000000000000000000..efa09d55236d7607a0a869eb7ab9ef4d6254231f
--- /dev/null
+++ b/lib/python3.1/site-packages/typing_extensions.py
@@ -0,0 +1,4244 @@
+import abc
+import builtins
+import collections
+import collections.abc
+import contextlib
+import enum
+import functools
+import inspect
+import io
+import keyword
+import operator
+import sys
+import types as _types
+import typing
+import warnings
+
+if sys.version_info >= (3, 14):
+    import annotationlib
+
+__all__ = [
+    # Super-special typing primitives.
+    'Any',
+    'ClassVar',
+    'Concatenate',
+    'Final',
+    'LiteralString',
+    'ParamSpec',
+    'ParamSpecArgs',
+    'ParamSpecKwargs',
+    'Self',
+    'Type',
+    'TypeVar',
+    'TypeVarTuple',
+    'Unpack',
+
+    # ABCs (from collections.abc).
+    'Awaitable',
+    'AsyncIterator',
+    'AsyncIterable',
+    'Coroutine',
+    'AsyncGenerator',
+    'AsyncContextManager',
+    'Buffer',
+    'ChainMap',
+
+    # Concrete collection types.
+    'ContextManager',
+    'Counter',
+    'Deque',
+    'DefaultDict',
+    'NamedTuple',
+    'OrderedDict',
+    'TypedDict',
+
+    # Structural checks, a.k.a. protocols.
+    'SupportsAbs',
+    'SupportsBytes',
+    'SupportsComplex',
+    'SupportsFloat',
+    'SupportsIndex',
+    'SupportsInt',
+    'SupportsRound',
+    'Reader',
+    'Writer',
+
+    # One-off things.
+    'Annotated',
+    'assert_never',
+    'assert_type',
+    'clear_overloads',
+    'dataclass_transform',
+    'deprecated',
+    'Doc',
+    'evaluate_forward_ref',
+    'get_overloads',
+    'final',
+    'Format',
+    'get_annotations',
+    'get_args',
+    'get_origin',
+    'get_original_bases',
+    'get_protocol_members',
+    'get_type_hints',
+    'IntVar',
+    'is_protocol',
+    'is_typeddict',
+    'Literal',
+    'NewType',
+    'overload',
+    'override',
+    'Protocol',
+    'Sentinel',
+    'reveal_type',
+    'runtime',
+    'runtime_checkable',
+    'Text',
+    'TypeAlias',
+    'TypeAliasType',
+    'TypeForm',
+    'TypeGuard',
+    'TypeIs',
+    'TYPE_CHECKING',
+    'Never',
+    'NoReturn',
+    'ReadOnly',
+    'Required',
+    'NotRequired',
+    'NoDefault',
+    'NoExtraItems',
+
+    # Pure aliases, have always been in typing
+    'AbstractSet',
+    'AnyStr',
+    'BinaryIO',
+    'Callable',
+    'Collection',
+    'Container',
+    'Dict',
+    'ForwardRef',
+    'FrozenSet',
+    'Generator',
+    'Generic',
+    'Hashable',
+    'IO',
+    'ItemsView',
+    'Iterable',
+    'Iterator',
+    'KeysView',
+    'List',
+    'Mapping',
+    'MappingView',
+    'Match',
+    'MutableMapping',
+    'MutableSequence',
+    'MutableSet',
+    'Optional',
+    'Pattern',
+    'Reversible',
+    'Sequence',
+    'Set',
+    'Sized',
+    'TextIO',
+    'Tuple',
+    'Union',
+    'ValuesView',
+    'cast',
+    'no_type_check',
+    'no_type_check_decorator',
+]
+
+# for backward compatibility
+PEP_560 = True
+GenericMeta = type
+_PEP_696_IMPLEMENTED = sys.version_info >= (3, 13, 0, "beta")
+
+# Added with bpo-45166 to 3.10.1+ and some 3.9 versions
+_FORWARD_REF_HAS_CLASS = "__forward_is_class__" in typing.ForwardRef.__slots__
+
+# The functions below are modified copies of typing internal helpers.
+# They are needed by _ProtocolMeta and they provide support for PEP 646.
+
+
+class _Sentinel:
+    def __repr__(self):
+        return ""
+
+
+_marker = _Sentinel()
+
+
+if sys.version_info >= (3, 10):
+    def _should_collect_from_parameters(t):
+        return isinstance(
+            t, (typing._GenericAlias, _types.GenericAlias, _types.UnionType)
+        )
+else:
+    def _should_collect_from_parameters(t):
+        return isinstance(t, (typing._GenericAlias, _types.GenericAlias))
+
+
+NoReturn = typing.NoReturn
+
+# Some unconstrained type variables.  These are used by the container types.
+# (These are not for export.)
+T = typing.TypeVar('T')  # Any type.
+KT = typing.TypeVar('KT')  # Key type.
+VT = typing.TypeVar('VT')  # Value type.
+T_co = typing.TypeVar('T_co', covariant=True)  # Any type covariant containers.
+T_contra = typing.TypeVar('T_contra', contravariant=True)  # Ditto contravariant.
+
+
+if sys.version_info >= (3, 11):
+    from typing import Any
+else:
+
+    class _AnyMeta(type):
+        def __instancecheck__(self, obj):
+            if self is Any:
+                raise TypeError("typing_extensions.Any cannot be used with isinstance()")
+            return super().__instancecheck__(obj)
+
+        def __repr__(self):
+            if self is Any:
+                return "typing_extensions.Any"
+            return super().__repr__()
+
+    class Any(metaclass=_AnyMeta):
+        """Special type indicating an unconstrained type.
+        - Any is compatible with every type.
+        - Any assumed to have all methods.
+        - All values assumed to be instances of Any.
+        Note that all the above statements are true from the point of view of
+        static type checkers. At runtime, Any should not be used with instance
+        checks.
+        """
+        def __new__(cls, *args, **kwargs):
+            if cls is Any:
+                raise TypeError("Any cannot be instantiated")
+            return super().__new__(cls, *args, **kwargs)
+
+
+ClassVar = typing.ClassVar
+
+# Vendored from cpython typing._SpecialFrom
+# Having a separate class means that instances will not be rejected by
+# typing._type_check.
+class _SpecialForm(typing._Final, _root=True):
+    __slots__ = ('_name', '__doc__', '_getitem')
+
+    def __init__(self, getitem):
+        self._getitem = getitem
+        self._name = getitem.__name__
+        self.__doc__ = getitem.__doc__
+
+    def __getattr__(self, item):
+        if item in {'__name__', '__qualname__'}:
+            return self._name
+
+        raise AttributeError(item)
+
+    def __mro_entries__(self, bases):
+        raise TypeError(f"Cannot subclass {self!r}")
+
+    def __repr__(self):
+        return f'typing_extensions.{self._name}'
+
+    def __reduce__(self):
+        return self._name
+
+    def __call__(self, *args, **kwds):
+        raise TypeError(f"Cannot instantiate {self!r}")
+
+    def __or__(self, other):
+        return typing.Union[self, other]
+
+    def __ror__(self, other):
+        return typing.Union[other, self]
+
+    def __instancecheck__(self, obj):
+        raise TypeError(f"{self} cannot be used with isinstance()")
+
+    def __subclasscheck__(self, cls):
+        raise TypeError(f"{self} cannot be used with issubclass()")
+
+    @typing._tp_cache
+    def __getitem__(self, parameters):
+        return self._getitem(self, parameters)
+
+
+# Note that inheriting from this class means that the object will be
+# rejected by typing._type_check, so do not use it if the special form
+# is arguably valid as a type by itself.
+class _ExtensionsSpecialForm(typing._SpecialForm, _root=True):
+    def __repr__(self):
+        return 'typing_extensions.' + self._name
+
+
+Final = typing.Final
+
+if sys.version_info >= (3, 11):
+    final = typing.final
+else:
+    # @final exists in 3.8+, but we backport it for all versions
+    # before 3.11 to keep support for the __final__ attribute.
+    # See https://bugs.python.org/issue46342
+    def final(f):
+        """This decorator can be used to indicate to type checkers that
+        the decorated method cannot be overridden, and decorated class
+        cannot be subclassed. For example:
+
+            class Base:
+                @final
+                def done(self) -> None:
+                    ...
+            class Sub(Base):
+                def done(self) -> None:  # Error reported by type checker
+                    ...
+            @final
+            class Leaf:
+                ...
+            class Other(Leaf):  # Error reported by type checker
+                ...
+
+        There is no runtime checking of these properties. The decorator
+        sets the ``__final__`` attribute to ``True`` on the decorated object
+        to allow runtime introspection.
+        """
+        try:
+            f.__final__ = True
+        except (AttributeError, TypeError):
+            # Skip the attribute silently if it is not writable.
+            # AttributeError happens if the object has __slots__ or a
+            # read-only property, TypeError if it's a builtin class.
+            pass
+        return f
+
+
+def IntVar(name):
+    return typing.TypeVar(name)
+
+
+# A Literal bug was fixed in 3.11.0, 3.10.1 and 3.9.8
+if sys.version_info >= (3, 10, 1):
+    Literal = typing.Literal
+else:
+    def _flatten_literal_params(parameters):
+        """An internal helper for Literal creation: flatten Literals among parameters"""
+        params = []
+        for p in parameters:
+            if isinstance(p, _LiteralGenericAlias):
+                params.extend(p.__args__)
+            else:
+                params.append(p)
+        return tuple(params)
+
+    def _value_and_type_iter(params):
+        for p in params:
+            yield p, type(p)
+
+    class _LiteralGenericAlias(typing._GenericAlias, _root=True):
+        def __eq__(self, other):
+            if not isinstance(other, _LiteralGenericAlias):
+                return NotImplemented
+            these_args_deduped = set(_value_and_type_iter(self.__args__))
+            other_args_deduped = set(_value_and_type_iter(other.__args__))
+            return these_args_deduped == other_args_deduped
+
+        def __hash__(self):
+            return hash(frozenset(_value_and_type_iter(self.__args__)))
+
+    class _LiteralForm(_ExtensionsSpecialForm, _root=True):
+        def __init__(self, doc: str):
+            self._name = 'Literal'
+            self._doc = self.__doc__ = doc
+
+        def __getitem__(self, parameters):
+            if not isinstance(parameters, tuple):
+                parameters = (parameters,)
+
+            parameters = _flatten_literal_params(parameters)
+
+            val_type_pairs = list(_value_and_type_iter(parameters))
+            try:
+                deduped_pairs = set(val_type_pairs)
+            except TypeError:
+                # unhashable parameters
+                pass
+            else:
+                # similar logic to typing._deduplicate on Python 3.9+
+                if len(deduped_pairs) < len(val_type_pairs):
+                    new_parameters = []
+                    for pair in val_type_pairs:
+                        if pair in deduped_pairs:
+                            new_parameters.append(pair[0])
+                            deduped_pairs.remove(pair)
+                    assert not deduped_pairs, deduped_pairs
+                    parameters = tuple(new_parameters)
+
+            return _LiteralGenericAlias(self, parameters)
+
+    Literal = _LiteralForm(doc="""\
+                           A type that can be used to indicate to type checkers
+                           that the corresponding value has a value literally equivalent
+                           to the provided parameter. For example:
+
+                               var: Literal[4] = 4
+
+                           The type checker understands that 'var' is literally equal to
+                           the value 4 and no other value.
+
+                           Literal[...] cannot be subclassed. There is no runtime
+                           checking verifying that the parameter is actually a value
+                           instead of a type.""")
+
+
+_overload_dummy = typing._overload_dummy
+
+
+if hasattr(typing, "get_overloads"):  # 3.11+
+    overload = typing.overload
+    get_overloads = typing.get_overloads
+    clear_overloads = typing.clear_overloads
+else:
+    # {module: {qualname: {firstlineno: func}}}
+    _overload_registry = collections.defaultdict(
+        functools.partial(collections.defaultdict, dict)
+    )
+
+    def overload(func):
+        """Decorator for overloaded functions/methods.
+
+        In a stub file, place two or more stub definitions for the same
+        function in a row, each decorated with @overload.  For example:
+
+        @overload
+        def utf8(value: None) -> None: ...
+        @overload
+        def utf8(value: bytes) -> bytes: ...
+        @overload
+        def utf8(value: str) -> bytes: ...
+
+        In a non-stub file (i.e. a regular .py file), do the same but
+        follow it with an implementation.  The implementation should *not*
+        be decorated with @overload.  For example:
+
+        @overload
+        def utf8(value: None) -> None: ...
+        @overload
+        def utf8(value: bytes) -> bytes: ...
+        @overload
+        def utf8(value: str) -> bytes: ...
+        def utf8(value):
+            # implementation goes here
+
+        The overloads for a function can be retrieved at runtime using the
+        get_overloads() function.
+        """
+        # classmethod and staticmethod
+        f = getattr(func, "__func__", func)
+        try:
+            _overload_registry[f.__module__][f.__qualname__][
+                f.__code__.co_firstlineno
+            ] = func
+        except AttributeError:
+            # Not a normal function; ignore.
+            pass
+        return _overload_dummy
+
+    def get_overloads(func):
+        """Return all defined overloads for *func* as a sequence."""
+        # classmethod and staticmethod
+        f = getattr(func, "__func__", func)
+        if f.__module__ not in _overload_registry:
+            return []
+        mod_dict = _overload_registry[f.__module__]
+        if f.__qualname__ not in mod_dict:
+            return []
+        return list(mod_dict[f.__qualname__].values())
+
+    def clear_overloads():
+        """Clear all overloads in the registry."""
+        _overload_registry.clear()
+
+
+# This is not a real generic class.  Don't use outside annotations.
+Type = typing.Type
+
+# Various ABCs mimicking those in collections.abc.
+# A few are simply re-exported for completeness.
+Awaitable = typing.Awaitable
+Coroutine = typing.Coroutine
+AsyncIterable = typing.AsyncIterable
+AsyncIterator = typing.AsyncIterator
+Deque = typing.Deque
+DefaultDict = typing.DefaultDict
+OrderedDict = typing.OrderedDict
+Counter = typing.Counter
+ChainMap = typing.ChainMap
+Text = typing.Text
+TYPE_CHECKING = typing.TYPE_CHECKING
+
+
+if sys.version_info >= (3, 13, 0, "beta"):
+    from typing import AsyncContextManager, AsyncGenerator, ContextManager, Generator
+else:
+    def _is_dunder(attr):
+        return attr.startswith('__') and attr.endswith('__')
+
+
+    class _SpecialGenericAlias(typing._SpecialGenericAlias, _root=True):
+        def __init__(self, origin, nparams, *, inst=True, name=None, defaults=()):
+            super().__init__(origin, nparams, inst=inst, name=name)
+            self._defaults = defaults
+
+        def __setattr__(self, attr, val):
+            allowed_attrs = {'_name', '_inst', '_nparams', '_defaults'}
+            if _is_dunder(attr) or attr in allowed_attrs:
+                object.__setattr__(self, attr, val)
+            else:
+                setattr(self.__origin__, attr, val)
+
+        @typing._tp_cache
+        def __getitem__(self, params):
+            if not isinstance(params, tuple):
+                params = (params,)
+            msg = "Parameters to generic types must be types."
+            params = tuple(typing._type_check(p, msg) for p in params)
+            if (
+                self._defaults
+                and len(params) < self._nparams
+                and len(params) + len(self._defaults) >= self._nparams
+            ):
+                params = (*params, *self._defaults[len(params) - self._nparams:])
+            actual_len = len(params)
+
+            if actual_len != self._nparams:
+                if self._defaults:
+                    expected = f"at least {self._nparams - len(self._defaults)}"
+                else:
+                    expected = str(self._nparams)
+                if not self._nparams:
+                    raise TypeError(f"{self} is not a generic class")
+                raise TypeError(
+                    f"Too {'many' if actual_len > self._nparams else 'few'}"
+                    f" arguments for {self};"
+                    f" actual {actual_len}, expected {expected}"
+                )
+            return self.copy_with(params)
+
+    _NoneType = type(None)
+    Generator = _SpecialGenericAlias(
+        collections.abc.Generator, 3, defaults=(_NoneType, _NoneType)
+    )
+    AsyncGenerator = _SpecialGenericAlias(
+        collections.abc.AsyncGenerator, 2, defaults=(_NoneType,)
+    )
+    ContextManager = _SpecialGenericAlias(
+        contextlib.AbstractContextManager,
+        2,
+        name="ContextManager",
+        defaults=(typing.Optional[bool],)
+    )
+    AsyncContextManager = _SpecialGenericAlias(
+        contextlib.AbstractAsyncContextManager,
+        2,
+        name="AsyncContextManager",
+        defaults=(typing.Optional[bool],)
+    )
+
+
+_PROTO_ALLOWLIST = {
+    'collections.abc': [
+        'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable',
+        'Hashable', 'Sized', 'Container', 'Collection', 'Reversible', 'Buffer',
+    ],
+    'contextlib': ['AbstractContextManager', 'AbstractAsyncContextManager'],
+    'typing_extensions': ['Buffer'],
+}
+
+
+_EXCLUDED_ATTRS = frozenset(typing.EXCLUDED_ATTRIBUTES) | {
+    "__match_args__", "__protocol_attrs__", "__non_callable_proto_members__",
+    "__final__",
+}
+
+
+def _get_protocol_attrs(cls):
+    attrs = set()
+    for base in cls.__mro__[:-1]:  # without object
+        if base.__name__ in {'Protocol', 'Generic'}:
+            continue
+        annotations = getattr(base, '__annotations__', {})
+        for attr in (*base.__dict__, *annotations):
+            if (not attr.startswith('_abc_') and attr not in _EXCLUDED_ATTRS):
+                attrs.add(attr)
+    return attrs
+
+
+def _caller(depth=1, default='__main__'):
+    try:
+        return sys._getframemodulename(depth + 1) or default
+    except AttributeError:  # For platforms without _getframemodulename()
+        pass
+    try:
+        return sys._getframe(depth + 1).f_globals.get('__name__', default)
+    except (AttributeError, ValueError):  # For platforms without _getframe()
+        pass
+    return None
+
+
+# `__match_args__` attribute was removed from protocol members in 3.13,
+# we want to backport this change to older Python versions.
+if sys.version_info >= (3, 13):
+    Protocol = typing.Protocol
+else:
+    def _allow_reckless_class_checks(depth=2):
+        """Allow instance and class checks for special stdlib modules.
+        The abc and functools modules indiscriminately call isinstance() and
+        issubclass() on the whole MRO of a user class, which may contain protocols.
+        """
+        return _caller(depth) in {'abc', 'functools', None}
+
+    def _no_init(self, *args, **kwargs):
+        if type(self)._is_protocol:
+            raise TypeError('Protocols cannot be instantiated')
+
+    def _type_check_issubclass_arg_1(arg):
+        """Raise TypeError if `arg` is not an instance of `type`
+        in `issubclass(arg, )`.
+
+        In most cases, this is verified by type.__subclasscheck__.
+        Checking it again unnecessarily would slow down issubclass() checks,
+        so, we don't perform this check unless we absolutely have to.
+
+        For various error paths, however,
+        we want to ensure that *this* error message is shown to the user
+        where relevant, rather than a typing.py-specific error message.
+        """
+        if not isinstance(arg, type):
+            # Same error message as for issubclass(1, int).
+            raise TypeError('issubclass() arg 1 must be a class')
+
+    # Inheriting from typing._ProtocolMeta isn't actually desirable,
+    # but is necessary to allow typing.Protocol and typing_extensions.Protocol
+    # to mix without getting TypeErrors about "metaclass conflict"
+    class _ProtocolMeta(type(typing.Protocol)):
+        # This metaclass is somewhat unfortunate,
+        # but is necessary for several reasons...
+        #
+        # NOTE: DO NOT call super() in any methods in this class
+        # That would call the methods on typing._ProtocolMeta on Python <=3.11
+        # and those are slow
+        def __new__(mcls, name, bases, namespace, **kwargs):
+            if name == "Protocol" and len(bases) < 2:
+                pass
+            elif {Protocol, typing.Protocol} & set(bases):
+                for base in bases:
+                    if not (
+                        base in {object, typing.Generic, Protocol, typing.Protocol}
+                        or base.__name__ in _PROTO_ALLOWLIST.get(base.__module__, [])
+                        or is_protocol(base)
+                    ):
+                        raise TypeError(
+                            f"Protocols can only inherit from other protocols, "
+                            f"got {base!r}"
+                        )
+            return abc.ABCMeta.__new__(mcls, name, bases, namespace, **kwargs)
+
+        def __init__(cls, *args, **kwargs):
+            abc.ABCMeta.__init__(cls, *args, **kwargs)
+            if getattr(cls, "_is_protocol", False):
+                cls.__protocol_attrs__ = _get_protocol_attrs(cls)
+
+        def __subclasscheck__(cls, other):
+            if cls is Protocol:
+                return type.__subclasscheck__(cls, other)
+            if (
+                getattr(cls, '_is_protocol', False)
+                and not _allow_reckless_class_checks()
+            ):
+                if not getattr(cls, '_is_runtime_protocol', False):
+                    _type_check_issubclass_arg_1(other)
+                    raise TypeError(
+                        "Instance and class checks can only be used with "
+                        "@runtime_checkable protocols"
+                    )
+                if (
+                    # this attribute is set by @runtime_checkable:
+                    cls.__non_callable_proto_members__
+                    and cls.__dict__.get("__subclasshook__") is _proto_hook
+                ):
+                    _type_check_issubclass_arg_1(other)
+                    non_method_attrs = sorted(cls.__non_callable_proto_members__)
+                    raise TypeError(
+                        "Protocols with non-method members don't support issubclass()."
+                        f" Non-method members: {str(non_method_attrs)[1:-1]}."
+                    )
+            return abc.ABCMeta.__subclasscheck__(cls, other)
+
+        def __instancecheck__(cls, instance):
+            # We need this method for situations where attributes are
+            # assigned in __init__.
+            if cls is Protocol:
+                return type.__instancecheck__(cls, instance)
+            if not getattr(cls, "_is_protocol", False):
+                # i.e., it's a concrete subclass of a protocol
+                return abc.ABCMeta.__instancecheck__(cls, instance)
+
+            if (
+                not getattr(cls, '_is_runtime_protocol', False) and
+                not _allow_reckless_class_checks()
+            ):
+                raise TypeError("Instance and class checks can only be used with"
+                                " @runtime_checkable protocols")
+
+            if abc.ABCMeta.__instancecheck__(cls, instance):
+                return True
+
+            for attr in cls.__protocol_attrs__:
+                try:
+                    val = inspect.getattr_static(instance, attr)
+                except AttributeError:
+                    break
+                # this attribute is set by @runtime_checkable:
+                if val is None and attr not in cls.__non_callable_proto_members__:
+                    break
+            else:
+                return True
+
+            return False
+
+        def __eq__(cls, other):
+            # Hack so that typing.Generic.__class_getitem__
+            # treats typing_extensions.Protocol
+            # as equivalent to typing.Protocol
+            if abc.ABCMeta.__eq__(cls, other) is True:
+                return True
+            return cls is Protocol and other is typing.Protocol
+
+        # This has to be defined, or the abc-module cache
+        # complains about classes with this metaclass being unhashable,
+        # if we define only __eq__!
+        def __hash__(cls) -> int:
+            return type.__hash__(cls)
+
+    @classmethod
+    def _proto_hook(cls, other):
+        if not cls.__dict__.get('_is_protocol', False):
+            return NotImplemented
+
+        for attr in cls.__protocol_attrs__:
+            for base in other.__mro__:
+                # Check if the members appears in the class dictionary...
+                if attr in base.__dict__:
+                    if base.__dict__[attr] is None:
+                        return NotImplemented
+                    break
+
+                # ...or in annotations, if it is a sub-protocol.
+                annotations = getattr(base, '__annotations__', {})
+                if (
+                    isinstance(annotations, collections.abc.Mapping)
+                    and attr in annotations
+                    and is_protocol(other)
+                ):
+                    break
+            else:
+                return NotImplemented
+        return True
+
+    class Protocol(typing.Generic, metaclass=_ProtocolMeta):
+        __doc__ = typing.Protocol.__doc__
+        __slots__ = ()
+        _is_protocol = True
+        _is_runtime_protocol = False
+
+        def __init_subclass__(cls, *args, **kwargs):
+            super().__init_subclass__(*args, **kwargs)
+
+            # Determine if this is a protocol or a concrete subclass.
+            if not cls.__dict__.get('_is_protocol', False):
+                cls._is_protocol = any(b is Protocol for b in cls.__bases__)
+
+            # Set (or override) the protocol subclass hook.
+            if '__subclasshook__' not in cls.__dict__:
+                cls.__subclasshook__ = _proto_hook
+
+            # Prohibit instantiation for protocol classes
+            if cls._is_protocol and cls.__init__ is Protocol.__init__:
+                cls.__init__ = _no_init
+
+
+if sys.version_info >= (3, 13):
+    runtime_checkable = typing.runtime_checkable
+else:
+    def runtime_checkable(cls):
+        """Mark a protocol class as a runtime protocol.
+
+        Such protocol can be used with isinstance() and issubclass().
+        Raise TypeError if applied to a non-protocol class.
+        This allows a simple-minded structural check very similar to
+        one trick ponies in collections.abc such as Iterable.
+
+        For example::
+
+            @runtime_checkable
+            class Closable(Protocol):
+                def close(self): ...
+
+            assert isinstance(open('/some/file'), Closable)
+
+        Warning: this will check only the presence of the required methods,
+        not their type signatures!
+        """
+        if not issubclass(cls, typing.Generic) or not getattr(cls, '_is_protocol', False):
+            raise TypeError(f'@runtime_checkable can be only applied to protocol classes,'
+                            f' got {cls!r}')
+        cls._is_runtime_protocol = True
+
+        # typing.Protocol classes on <=3.11 break if we execute this block,
+        # because typing.Protocol classes on <=3.11 don't have a
+        # `__protocol_attrs__` attribute, and this block relies on the
+        # `__protocol_attrs__` attribute. Meanwhile, typing.Protocol classes on 3.12.2+
+        # break if we *don't* execute this block, because *they* assume that all
+        # protocol classes have a `__non_callable_proto_members__` attribute
+        # (which this block sets)
+        if isinstance(cls, _ProtocolMeta) or sys.version_info >= (3, 12, 2):
+            # PEP 544 prohibits using issubclass()
+            # with protocols that have non-method members.
+            # See gh-113320 for why we compute this attribute here,
+            # rather than in `_ProtocolMeta.__init__`
+            cls.__non_callable_proto_members__ = set()
+            for attr in cls.__protocol_attrs__:
+                try:
+                    is_callable = callable(getattr(cls, attr, None))
+                except Exception as e:
+                    raise TypeError(
+                        f"Failed to determine whether protocol member {attr!r} "
+                        "is a method member"
+                    ) from e
+                else:
+                    if not is_callable:
+                        cls.__non_callable_proto_members__.add(attr)
+
+        return cls
+
+
+# The "runtime" alias exists for backwards compatibility.
+runtime = runtime_checkable
+
+
+# Our version of runtime-checkable protocols is faster on Python <=3.11
+if sys.version_info >= (3, 12):
+    SupportsInt = typing.SupportsInt
+    SupportsFloat = typing.SupportsFloat
+    SupportsComplex = typing.SupportsComplex
+    SupportsBytes = typing.SupportsBytes
+    SupportsIndex = typing.SupportsIndex
+    SupportsAbs = typing.SupportsAbs
+    SupportsRound = typing.SupportsRound
+else:
+    @runtime_checkable
+    class SupportsInt(Protocol):
+        """An ABC with one abstract method __int__."""
+        __slots__ = ()
+
+        @abc.abstractmethod
+        def __int__(self) -> int:
+            pass
+
+    @runtime_checkable
+    class SupportsFloat(Protocol):
+        """An ABC with one abstract method __float__."""
+        __slots__ = ()
+
+        @abc.abstractmethod
+        def __float__(self) -> float:
+            pass
+
+    @runtime_checkable
+    class SupportsComplex(Protocol):
+        """An ABC with one abstract method __complex__."""
+        __slots__ = ()
+
+        @abc.abstractmethod
+        def __complex__(self) -> complex:
+            pass
+
+    @runtime_checkable
+    class SupportsBytes(Protocol):
+        """An ABC with one abstract method __bytes__."""
+        __slots__ = ()
+
+        @abc.abstractmethod
+        def __bytes__(self) -> bytes:
+            pass
+
+    @runtime_checkable
+    class SupportsIndex(Protocol):
+        __slots__ = ()
+
+        @abc.abstractmethod
+        def __index__(self) -> int:
+            pass
+
+    @runtime_checkable
+    class SupportsAbs(Protocol[T_co]):
+        """
+        An ABC with one abstract method __abs__ that is covariant in its return type.
+        """
+        __slots__ = ()
+
+        @abc.abstractmethod
+        def __abs__(self) -> T_co:
+            pass
+
+    @runtime_checkable
+    class SupportsRound(Protocol[T_co]):
+        """
+        An ABC with one abstract method __round__ that is covariant in its return type.
+        """
+        __slots__ = ()
+
+        @abc.abstractmethod
+        def __round__(self, ndigits: int = 0) -> T_co:
+            pass
+
+
+if hasattr(io, "Reader") and hasattr(io, "Writer"):
+    Reader = io.Reader
+    Writer = io.Writer
+else:
+    @runtime_checkable
+    class Reader(Protocol[T_co]):
+        """Protocol for simple I/O reader instances.
+
+        This protocol only supports blocking I/O.
+        """
+
+        __slots__ = ()
+
+        @abc.abstractmethod
+        def read(self, size: int = ..., /) -> T_co:
+            """Read data from the input stream and return it.
+
+            If *size* is specified, at most *size* items (bytes/characters) will be
+            read.
+            """
+
+    @runtime_checkable
+    class Writer(Protocol[T_contra]):
+        """Protocol for simple I/O writer instances.
+
+        This protocol only supports blocking I/O.
+        """
+
+        __slots__ = ()
+
+        @abc.abstractmethod
+        def write(self, data: T_contra, /) -> int:
+            """Write *data* to the output stream and return the number of items written."""  # noqa: E501
+
+
+_NEEDS_SINGLETONMETA = (
+    not hasattr(typing, "NoDefault") or not hasattr(typing, "NoExtraItems")
+)
+
+if _NEEDS_SINGLETONMETA:
+    class SingletonMeta(type):
+        def __setattr__(cls, attr, value):
+            # TypeError is consistent with the behavior of NoneType
+            raise TypeError(
+                f"cannot set {attr!r} attribute of immutable type {cls.__name__!r}"
+            )
+
+
+if hasattr(typing, "NoDefault"):
+    NoDefault = typing.NoDefault
+else:
+    class NoDefaultType(metaclass=SingletonMeta):
+        """The type of the NoDefault singleton."""
+
+        __slots__ = ()
+
+        def __new__(cls):
+            return globals().get("NoDefault") or object.__new__(cls)
+
+        def __repr__(self):
+            return "typing_extensions.NoDefault"
+
+        def __reduce__(self):
+            return "NoDefault"
+
+    NoDefault = NoDefaultType()
+    del NoDefaultType
+
+if hasattr(typing, "NoExtraItems"):
+    NoExtraItems = typing.NoExtraItems
+else:
+    class NoExtraItemsType(metaclass=SingletonMeta):
+        """The type of the NoExtraItems singleton."""
+
+        __slots__ = ()
+
+        def __new__(cls):
+            return globals().get("NoExtraItems") or object.__new__(cls)
+
+        def __repr__(self):
+            return "typing_extensions.NoExtraItems"
+
+        def __reduce__(self):
+            return "NoExtraItems"
+
+    NoExtraItems = NoExtraItemsType()
+    del NoExtraItemsType
+
+if _NEEDS_SINGLETONMETA:
+    del SingletonMeta
+
+
+# Update this to something like >=3.13.0b1 if and when
+# PEP 728 is implemented in CPython
+_PEP_728_IMPLEMENTED = False
+
+if _PEP_728_IMPLEMENTED:
+    # The standard library TypedDict in Python 3.9.0/1 does not honour the "total"
+    # keyword with old-style TypedDict().  See https://bugs.python.org/issue42059
+    # The standard library TypedDict below Python 3.11 does not store runtime
+    # information about optional and required keys when using Required or NotRequired.
+    # Generic TypedDicts are also impossible using typing.TypedDict on Python <3.11.
+    # Aaaand on 3.12 we add __orig_bases__ to TypedDict
+    # to enable better runtime introspection.
+    # On 3.13 we deprecate some odd ways of creating TypedDicts.
+    # Also on 3.13, PEP 705 adds the ReadOnly[] qualifier.
+    # PEP 728 (still pending) makes more changes.
+    TypedDict = typing.TypedDict
+    _TypedDictMeta = typing._TypedDictMeta
+    is_typeddict = typing.is_typeddict
+else:
+    # 3.10.0 and later
+    _TAKES_MODULE = "module" in inspect.signature(typing._type_check).parameters
+
+    def _get_typeddict_qualifiers(annotation_type):
+        while True:
+            annotation_origin = get_origin(annotation_type)
+            if annotation_origin is Annotated:
+                annotation_args = get_args(annotation_type)
+                if annotation_args:
+                    annotation_type = annotation_args[0]
+                else:
+                    break
+            elif annotation_origin is Required:
+                yield Required
+                annotation_type, = get_args(annotation_type)
+            elif annotation_origin is NotRequired:
+                yield NotRequired
+                annotation_type, = get_args(annotation_type)
+            elif annotation_origin is ReadOnly:
+                yield ReadOnly
+                annotation_type, = get_args(annotation_type)
+            else:
+                break
+
+    class _TypedDictMeta(type):
+
+        def __new__(cls, name, bases, ns, *, total=True, closed=None,
+                    extra_items=NoExtraItems):
+            """Create new typed dict class object.
+
+            This method is called when TypedDict is subclassed,
+            or when TypedDict is instantiated. This way
+            TypedDict supports all three syntax forms described in its docstring.
+            Subclasses and instances of TypedDict return actual dictionaries.
+            """
+            for base in bases:
+                if type(base) is not _TypedDictMeta and base is not typing.Generic:
+                    raise TypeError('cannot inherit from both a TypedDict type '
+                                    'and a non-TypedDict base class')
+            if closed is not None and extra_items is not NoExtraItems:
+                raise TypeError(f"Cannot combine closed={closed!r} and extra_items")
+
+            if any(issubclass(b, typing.Generic) for b in bases):
+                generic_base = (typing.Generic,)
+            else:
+                generic_base = ()
+
+            ns_annotations = ns.pop('__annotations__', None)
+
+            # typing.py generally doesn't let you inherit from plain Generic, unless
+            # the name of the class happens to be "Protocol"
+            tp_dict = type.__new__(_TypedDictMeta, "Protocol", (*generic_base, dict), ns)
+            tp_dict.__name__ = name
+            if tp_dict.__qualname__ == "Protocol":
+                tp_dict.__qualname__ = name
+
+            if not hasattr(tp_dict, '__orig_bases__'):
+                tp_dict.__orig_bases__ = bases
+
+            annotations = {}
+            own_annotate = None
+            if ns_annotations is not None:
+                own_annotations = ns_annotations
+            elif sys.version_info >= (3, 14):
+                if hasattr(annotationlib, "get_annotate_from_class_namespace"):
+                    own_annotate = annotationlib.get_annotate_from_class_namespace(ns)
+                else:
+                    # 3.14.0a7 and earlier
+                    own_annotate = ns.get("__annotate__")
+                if own_annotate is not None:
+                    own_annotations = annotationlib.call_annotate_function(
+                        own_annotate, Format.FORWARDREF, owner=tp_dict
+                    )
+                else:
+                    own_annotations = {}
+            else:
+                own_annotations = {}
+            msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type"
+            if _TAKES_MODULE:
+                own_checked_annotations = {
+                    n: typing._type_check(tp, msg, module=tp_dict.__module__)
+                    for n, tp in own_annotations.items()
+                }
+            else:
+                own_checked_annotations = {
+                    n: typing._type_check(tp, msg)
+                    for n, tp in own_annotations.items()
+                }
+            required_keys = set()
+            optional_keys = set()
+            readonly_keys = set()
+            mutable_keys = set()
+            extra_items_type = extra_items
+
+            for base in bases:
+                base_dict = base.__dict__
+
+                if sys.version_info <= (3, 14):
+                    annotations.update(base_dict.get('__annotations__', {}))
+                required_keys.update(base_dict.get('__required_keys__', ()))
+                optional_keys.update(base_dict.get('__optional_keys__', ()))
+                readonly_keys.update(base_dict.get('__readonly_keys__', ()))
+                mutable_keys.update(base_dict.get('__mutable_keys__', ()))
+
+            # This was specified in an earlier version of PEP 728. Support
+            # is retained for backwards compatibility, but only for Python
+            # 3.13 and lower.
+            if (closed and sys.version_info < (3, 14)
+                       and "__extra_items__" in own_checked_annotations):
+                annotation_type = own_checked_annotations.pop("__extra_items__")
+                qualifiers = set(_get_typeddict_qualifiers(annotation_type))
+                if Required in qualifiers:
+                    raise TypeError(
+                        "Special key __extra_items__ does not support "
+                        "Required"
+                    )
+                if NotRequired in qualifiers:
+                    raise TypeError(
+                        "Special key __extra_items__ does not support "
+                        "NotRequired"
+                    )
+                extra_items_type = annotation_type
+
+            annotations.update(own_checked_annotations)
+            for annotation_key, annotation_type in own_checked_annotations.items():
+                qualifiers = set(_get_typeddict_qualifiers(annotation_type))
+
+                if Required in qualifiers:
+                    required_keys.add(annotation_key)
+                elif NotRequired in qualifiers:
+                    optional_keys.add(annotation_key)
+                elif total:
+                    required_keys.add(annotation_key)
+                else:
+                    optional_keys.add(annotation_key)
+                if ReadOnly in qualifiers:
+                    mutable_keys.discard(annotation_key)
+                    readonly_keys.add(annotation_key)
+                else:
+                    mutable_keys.add(annotation_key)
+                    readonly_keys.discard(annotation_key)
+
+            if sys.version_info >= (3, 14):
+                def __annotate__(format):
+                    annos = {}
+                    for base in bases:
+                        if base is Generic:
+                            continue
+                        base_annotate = base.__annotate__
+                        if base_annotate is None:
+                            continue
+                        base_annos = annotationlib.call_annotate_function(
+                            base_annotate, format, owner=base)
+                        annos.update(base_annos)
+                    if own_annotate is not None:
+                        own = annotationlib.call_annotate_function(
+                            own_annotate, format, owner=tp_dict)
+                        if format != Format.STRING:
+                            own = {
+                                n: typing._type_check(tp, msg, module=tp_dict.__module__)
+                                for n, tp in own.items()
+                            }
+                    elif format == Format.STRING:
+                        own = annotationlib.annotations_to_string(own_annotations)
+                    elif format in (Format.FORWARDREF, Format.VALUE):
+                        own = own_checked_annotations
+                    else:
+                        raise NotImplementedError(format)
+                    annos.update(own)
+                    return annos
+
+                tp_dict.__annotate__ = __annotate__
+            else:
+                tp_dict.__annotations__ = annotations
+            tp_dict.__required_keys__ = frozenset(required_keys)
+            tp_dict.__optional_keys__ = frozenset(optional_keys)
+            tp_dict.__readonly_keys__ = frozenset(readonly_keys)
+            tp_dict.__mutable_keys__ = frozenset(mutable_keys)
+            tp_dict.__total__ = total
+            tp_dict.__closed__ = closed
+            tp_dict.__extra_items__ = extra_items_type
+            return tp_dict
+
+        __call__ = dict  # static method
+
+        def __subclasscheck__(cls, other):
+            # Typed dicts are only for static structural subtyping.
+            raise TypeError('TypedDict does not support instance and class checks')
+
+        __instancecheck__ = __subclasscheck__
+
+    _TypedDict = type.__new__(_TypedDictMeta, 'TypedDict', (), {})
+
+    def _create_typeddict(
+        typename,
+        fields,
+        /,
+        *,
+        typing_is_inline,
+        total,
+        closed,
+        extra_items,
+        **kwargs,
+    ):
+        if fields is _marker or fields is None:
+            if fields is _marker:
+                deprecated_thing = (
+                    "Failing to pass a value for the 'fields' parameter"
+                )
+            else:
+                deprecated_thing = "Passing `None` as the 'fields' parameter"
+
+            example = f"`{typename} = TypedDict({typename!r}, {{}})`"
+            deprecation_msg = (
+                f"{deprecated_thing} is deprecated and will be disallowed in "
+                "Python 3.15. To create a TypedDict class with 0 fields "
+                "using the functional syntax, pass an empty dictionary, e.g. "
+            ) + example + "."
+            warnings.warn(deprecation_msg, DeprecationWarning, stacklevel=2)
+            # Support a field called "closed"
+            if closed is not False and closed is not True and closed is not None:
+                kwargs["closed"] = closed
+                closed = None
+            # Or "extra_items"
+            if extra_items is not NoExtraItems:
+                kwargs["extra_items"] = extra_items
+                extra_items = NoExtraItems
+            fields = kwargs
+        elif kwargs:
+            raise TypeError("TypedDict takes either a dict or keyword arguments,"
+                            " but not both")
+        if kwargs:
+            if sys.version_info >= (3, 13):
+                raise TypeError("TypedDict takes no keyword arguments")
+            warnings.warn(
+                "The kwargs-based syntax for TypedDict definitions is deprecated "
+                "in Python 3.11, will be removed in Python 3.13, and may not be "
+                "understood by third-party type checkers.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
+        ns = {'__annotations__': dict(fields)}
+        module = _caller(depth=4 if typing_is_inline else 2)
+        if module is not None:
+            # Setting correct module is necessary to make typed dict classes
+            # pickleable.
+            ns['__module__'] = module
+
+        td = _TypedDictMeta(typename, (), ns, total=total, closed=closed,
+                            extra_items=extra_items)
+        td.__orig_bases__ = (TypedDict,)
+        return td
+
+    class _TypedDictSpecialForm(_SpecialForm, _root=True):
+        def __call__(
+            self,
+            typename,
+            fields=_marker,
+            /,
+            *,
+            total=True,
+            closed=None,
+            extra_items=NoExtraItems,
+            **kwargs
+        ):
+            return _create_typeddict(
+                typename,
+                fields,
+                typing_is_inline=False,
+                total=total,
+                closed=closed,
+                extra_items=extra_items,
+                **kwargs,
+            )
+
+        def __mro_entries__(self, bases):
+            return (_TypedDict,)
+
+    @_TypedDictSpecialForm
+    def TypedDict(self, args):
+        """A simple typed namespace. At runtime it is equivalent to a plain dict.
+
+        TypedDict creates a dictionary type such that a type checker will expect all
+        instances to have a certain set of keys, where each key is
+        associated with a value of a consistent type. This expectation
+        is not checked at runtime.
+
+        Usage::
+
+            class Point2D(TypedDict):
+                x: int
+                y: int
+                label: str
+
+            a: Point2D = {'x': 1, 'y': 2, 'label': 'good'}  # OK
+            b: Point2D = {'z': 3, 'label': 'bad'}           # Fails type check
+
+            assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first')
+
+        The type info can be accessed via the Point2D.__annotations__ dict, and
+        the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets.
+        TypedDict supports an additional equivalent form::
+
+            Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str})
+
+        By default, all keys must be present in a TypedDict. It is possible
+        to override this by specifying totality::
+
+            class Point2D(TypedDict, total=False):
+                x: int
+                y: int
+
+        This means that a Point2D TypedDict can have any of the keys omitted. A type
+        checker is only expected to support a literal False or True as the value of
+        the total argument. True is the default, and makes all items defined in the
+        class body be required.
+
+        The Required and NotRequired special forms can also be used to mark
+        individual keys as being required or not required::
+
+            class Point2D(TypedDict):
+                x: int  # the "x" key must always be present (Required is the default)
+                y: NotRequired[int]  # the "y" key can be omitted
+
+        See PEP 655 for more details on Required and NotRequired.
+        """
+        # This runs when creating inline TypedDicts:
+        if not isinstance(args, dict):
+            raise TypeError(
+                "TypedDict[...] should be used with a single dict argument"
+            )
+
+        return _create_typeddict(
+            "",
+            args,
+            typing_is_inline=True,
+            total=True,
+            closed=True,
+            extra_items=NoExtraItems,
+        )
+
+    _TYPEDDICT_TYPES = (typing._TypedDictMeta, _TypedDictMeta)
+
+    def is_typeddict(tp):
+        """Check if an annotation is a TypedDict class
+
+        For example::
+            class Film(TypedDict):
+                title: str
+                year: int
+
+            is_typeddict(Film)  # => True
+            is_typeddict(Union[list, str])  # => False
+        """
+        return isinstance(tp, _TYPEDDICT_TYPES)
+
+
+if hasattr(typing, "assert_type"):
+    assert_type = typing.assert_type
+
+else:
+    def assert_type(val, typ, /):
+        """Assert (to the type checker) that the value is of the given type.
+
+        When the type checker encounters a call to assert_type(), it
+        emits an error if the value is not of the specified type::
+
+            def greet(name: str) -> None:
+                assert_type(name, str)  # ok
+                assert_type(name, int)  # type checker error
+
+        At runtime this returns the first argument unchanged and otherwise
+        does nothing.
+        """
+        return val
+
+
+if hasattr(typing, "ReadOnly"):  # 3.13+
+    get_type_hints = typing.get_type_hints
+else:  # <=3.13
+    # replaces _strip_annotations()
+    def _strip_extras(t):
+        """Strips Annotated, Required and NotRequired from a given type."""
+        if isinstance(t, typing._AnnotatedAlias):
+            return _strip_extras(t.__origin__)
+        if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired, ReadOnly):
+            return _strip_extras(t.__args__[0])
+        if isinstance(t, typing._GenericAlias):
+            stripped_args = tuple(_strip_extras(a) for a in t.__args__)
+            if stripped_args == t.__args__:
+                return t
+            return t.copy_with(stripped_args)
+        if hasattr(_types, "GenericAlias") and isinstance(t, _types.GenericAlias):
+            stripped_args = tuple(_strip_extras(a) for a in t.__args__)
+            if stripped_args == t.__args__:
+                return t
+            return _types.GenericAlias(t.__origin__, stripped_args)
+        if hasattr(_types, "UnionType") and isinstance(t, _types.UnionType):
+            stripped_args = tuple(_strip_extras(a) for a in t.__args__)
+            if stripped_args == t.__args__:
+                return t
+            return functools.reduce(operator.or_, stripped_args)
+
+        return t
+
+    def get_type_hints(obj, globalns=None, localns=None, include_extras=False):
+        """Return type hints for an object.
+
+        This is often the same as obj.__annotations__, but it handles
+        forward references encoded as string literals, adds Optional[t] if a
+        default value equal to None is set and recursively replaces all
+        'Annotated[T, ...]', 'Required[T]' or 'NotRequired[T]' with 'T'
+        (unless 'include_extras=True').
+
+        The argument may be a module, class, method, or function. The annotations
+        are returned as a dictionary. For classes, annotations include also
+        inherited members.
+
+        TypeError is raised if the argument is not of a type that can contain
+        annotations, and an empty dictionary is returned if no annotations are
+        present.
+
+        BEWARE -- the behavior of globalns and localns is counterintuitive
+        (unless you are familiar with how eval() and exec() work).  The
+        search order is locals first, then globals.
+
+        - If no dict arguments are passed, an attempt is made to use the
+          globals from obj (or the respective module's globals for classes),
+          and these are also used as the locals.  If the object does not appear
+          to have globals, an empty dictionary is used.
+
+        - If one dict argument is passed, it is used for both globals and
+          locals.
+
+        - If two dict arguments are passed, they specify globals and
+          locals, respectively.
+        """
+        hint = typing.get_type_hints(
+            obj, globalns=globalns, localns=localns, include_extras=True
+        )
+        if sys.version_info < (3, 11):
+            _clean_optional(obj, hint, globalns, localns)
+        if include_extras:
+            return hint
+        return {k: _strip_extras(t) for k, t in hint.items()}
+
+    _NoneType = type(None)
+
+    def _could_be_inserted_optional(t):
+        """detects Union[..., None] pattern"""
+        if not isinstance(t, typing._UnionGenericAlias):
+            return False
+        # Assume if last argument is not None they are user defined
+        if t.__args__[-1] is not _NoneType:
+            return False
+        return True
+
+    # < 3.11
+    def _clean_optional(obj, hints, globalns=None, localns=None):
+        # reverts injected Union[..., None] cases from typing.get_type_hints
+        # when a None default value is used.
+        # see https://github.com/python/typing_extensions/issues/310
+        if not hints or isinstance(obj, type):
+            return
+        defaults = typing._get_defaults(obj)  # avoid accessing __annotations___
+        if not defaults:
+            return
+        original_hints = obj.__annotations__
+        for name, value in hints.items():
+            # Not a Union[..., None] or replacement conditions not fullfilled
+            if (not _could_be_inserted_optional(value)
+                or name not in defaults
+                or defaults[name] is not None
+            ):
+                continue
+            original_value = original_hints[name]
+            # value=NoneType should have caused a skip above but check for safety
+            if original_value is None:
+                original_value = _NoneType
+            # Forward reference
+            if isinstance(original_value, str):
+                if globalns is None:
+                    if isinstance(obj, _types.ModuleType):
+                        globalns = obj.__dict__
+                    else:
+                        nsobj = obj
+                        # Find globalns for the unwrapped object.
+                        while hasattr(nsobj, '__wrapped__'):
+                            nsobj = nsobj.__wrapped__
+                        globalns = getattr(nsobj, '__globals__', {})
+                    if localns is None:
+                        localns = globalns
+                elif localns is None:
+                    localns = globalns
+
+                original_value = ForwardRef(
+                    original_value,
+                    is_argument=not isinstance(obj, _types.ModuleType)
+                )
+            original_evaluated = typing._eval_type(original_value, globalns, localns)
+            # Compare if values differ. Note that even if equal
+            # value might be cached by typing._tp_cache contrary to original_evaluated
+            if original_evaluated != value or (
+                # 3.10: ForwardRefs of UnionType might be turned into _UnionGenericAlias
+                hasattr(_types, "UnionType")
+                and isinstance(original_evaluated, _types.UnionType)
+                and not isinstance(value, _types.UnionType)
+            ):
+                hints[name] = original_evaluated
+
+# Python 3.9 has get_origin() and get_args() but those implementations don't support
+# ParamSpecArgs and ParamSpecKwargs, so only Python 3.10's versions will do.
+if sys.version_info[:2] >= (3, 10):
+    get_origin = typing.get_origin
+    get_args = typing.get_args
+# 3.9
+else:
+    def get_origin(tp):
+        """Get the unsubscripted version of a type.
+
+        This supports generic types, Callable, Tuple, Union, Literal, Final, ClassVar
+        and Annotated. Return None for unsupported types. Examples::
+
+            get_origin(Literal[42]) is Literal
+            get_origin(int) is None
+            get_origin(ClassVar[int]) is ClassVar
+            get_origin(Generic) is Generic
+            get_origin(Generic[T]) is Generic
+            get_origin(Union[T, int]) is Union
+            get_origin(List[Tuple[T, T]][int]) == list
+            get_origin(P.args) is P
+        """
+        if isinstance(tp, typing._AnnotatedAlias):
+            return Annotated
+        if isinstance(tp, (typing._BaseGenericAlias, _types.GenericAlias,
+                           ParamSpecArgs, ParamSpecKwargs)):
+            return tp.__origin__
+        if tp is typing.Generic:
+            return typing.Generic
+        return None
+
+    def get_args(tp):
+        """Get type arguments with all substitutions performed.
+
+        For unions, basic simplifications used by Union constructor are performed.
+        Examples::
+            get_args(Dict[str, int]) == (str, int)
+            get_args(int) == ()
+            get_args(Union[int, Union[T, int], str][int]) == (int, str)
+            get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int])
+            get_args(Callable[[], T][int]) == ([], int)
+        """
+        if isinstance(tp, typing._AnnotatedAlias):
+            return (tp.__origin__, *tp.__metadata__)
+        if isinstance(tp, (typing._GenericAlias, _types.GenericAlias)):
+            res = tp.__args__
+            if get_origin(tp) is collections.abc.Callable and res[0] is not Ellipsis:
+                res = (list(res[:-1]), res[-1])
+            return res
+        return ()
+
+
+# 3.10+
+if hasattr(typing, 'TypeAlias'):
+    TypeAlias = typing.TypeAlias
+# 3.9
+else:
+    @_ExtensionsSpecialForm
+    def TypeAlias(self, parameters):
+        """Special marker indicating that an assignment should
+        be recognized as a proper type alias definition by type
+        checkers.
+
+        For example::
+
+            Predicate: TypeAlias = Callable[..., bool]
+
+        It's invalid when used anywhere except as in the example above.
+        """
+        raise TypeError(f"{self} is not subscriptable")
+
+
+def _set_default(type_param, default):
+    type_param.has_default = lambda: default is not NoDefault
+    type_param.__default__ = default
+
+
+def _set_module(typevarlike):
+    # for pickling:
+    def_mod = _caller(depth=2)
+    if def_mod != 'typing_extensions':
+        typevarlike.__module__ = def_mod
+
+
+class _DefaultMixin:
+    """Mixin for TypeVarLike defaults."""
+
+    __slots__ = ()
+    __init__ = _set_default
+
+
+# Classes using this metaclass must provide a _backported_typevarlike ClassVar
+class _TypeVarLikeMeta(type):
+    def __instancecheck__(cls, __instance: Any) -> bool:
+        return isinstance(__instance, cls._backported_typevarlike)
+
+
+if _PEP_696_IMPLEMENTED:
+    from typing import TypeVar
+else:
+    # Add default and infer_variance parameters from PEP 696 and 695
+    class TypeVar(metaclass=_TypeVarLikeMeta):
+        """Type variable."""
+
+        _backported_typevarlike = typing.TypeVar
+
+        def __new__(cls, name, *constraints, bound=None,
+                    covariant=False, contravariant=False,
+                    default=NoDefault, infer_variance=False):
+            if hasattr(typing, "TypeAliasType"):
+                # PEP 695 implemented (3.12+), can pass infer_variance to typing.TypeVar
+                typevar = typing.TypeVar(name, *constraints, bound=bound,
+                                         covariant=covariant, contravariant=contravariant,
+                                         infer_variance=infer_variance)
+            else:
+                typevar = typing.TypeVar(name, *constraints, bound=bound,
+                                         covariant=covariant, contravariant=contravariant)
+                if infer_variance and (covariant or contravariant):
+                    raise ValueError("Variance cannot be specified with infer_variance.")
+                typevar.__infer_variance__ = infer_variance
+
+            _set_default(typevar, default)
+            _set_module(typevar)
+
+            def _tvar_prepare_subst(alias, args):
+                if (
+                    typevar.has_default()
+                    and alias.__parameters__.index(typevar) == len(args)
+                ):
+                    args += (typevar.__default__,)
+                return args
+
+            typevar.__typing_prepare_subst__ = _tvar_prepare_subst
+            return typevar
+
+        def __init_subclass__(cls) -> None:
+            raise TypeError(f"type '{__name__}.TypeVar' is not an acceptable base type")
+
+
+# Python 3.10+ has PEP 612
+if hasattr(typing, 'ParamSpecArgs'):
+    ParamSpecArgs = typing.ParamSpecArgs
+    ParamSpecKwargs = typing.ParamSpecKwargs
+# 3.9
+else:
+    class _Immutable:
+        """Mixin to indicate that object should not be copied."""
+        __slots__ = ()
+
+        def __copy__(self):
+            return self
+
+        def __deepcopy__(self, memo):
+            return self
+
+    class ParamSpecArgs(_Immutable):
+        """The args for a ParamSpec object.
+
+        Given a ParamSpec object P, P.args is an instance of ParamSpecArgs.
+
+        ParamSpecArgs objects have a reference back to their ParamSpec:
+
+        P.args.__origin__ is P
+
+        This type is meant for runtime introspection and has no special meaning to
+        static type checkers.
+        """
+        def __init__(self, origin):
+            self.__origin__ = origin
+
+        def __repr__(self):
+            return f"{self.__origin__.__name__}.args"
+
+        def __eq__(self, other):
+            if not isinstance(other, ParamSpecArgs):
+                return NotImplemented
+            return self.__origin__ == other.__origin__
+
+    class ParamSpecKwargs(_Immutable):
+        """The kwargs for a ParamSpec object.
+
+        Given a ParamSpec object P, P.kwargs is an instance of ParamSpecKwargs.
+
+        ParamSpecKwargs objects have a reference back to their ParamSpec:
+
+        P.kwargs.__origin__ is P
+
+        This type is meant for runtime introspection and has no special meaning to
+        static type checkers.
+        """
+        def __init__(self, origin):
+            self.__origin__ = origin
+
+        def __repr__(self):
+            return f"{self.__origin__.__name__}.kwargs"
+
+        def __eq__(self, other):
+            if not isinstance(other, ParamSpecKwargs):
+                return NotImplemented
+            return self.__origin__ == other.__origin__
+
+
+if _PEP_696_IMPLEMENTED:
+    from typing import ParamSpec
+
+# 3.10+
+elif hasattr(typing, 'ParamSpec'):
+
+    # Add default parameter - PEP 696
+    class ParamSpec(metaclass=_TypeVarLikeMeta):
+        """Parameter specification."""
+
+        _backported_typevarlike = typing.ParamSpec
+
+        def __new__(cls, name, *, bound=None,
+                    covariant=False, contravariant=False,
+                    infer_variance=False, default=NoDefault):
+            if hasattr(typing, "TypeAliasType"):
+                # PEP 695 implemented, can pass infer_variance to typing.TypeVar
+                paramspec = typing.ParamSpec(name, bound=bound,
+                                             covariant=covariant,
+                                             contravariant=contravariant,
+                                             infer_variance=infer_variance)
+            else:
+                paramspec = typing.ParamSpec(name, bound=bound,
+                                             covariant=covariant,
+                                             contravariant=contravariant)
+                paramspec.__infer_variance__ = infer_variance
+
+            _set_default(paramspec, default)
+            _set_module(paramspec)
+
+            def _paramspec_prepare_subst(alias, args):
+                params = alias.__parameters__
+                i = params.index(paramspec)
+                if i == len(args) and paramspec.has_default():
+                    args = [*args, paramspec.__default__]
+                if i >= len(args):
+                    raise TypeError(f"Too few arguments for {alias}")
+                # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612.
+                if len(params) == 1 and not typing._is_param_expr(args[0]):
+                    assert i == 0
+                    args = (args,)
+                # Convert lists to tuples to help other libraries cache the results.
+                elif isinstance(args[i], list):
+                    args = (*args[:i], tuple(args[i]), *args[i + 1:])
+                return args
+
+            paramspec.__typing_prepare_subst__ = _paramspec_prepare_subst
+            return paramspec
+
+        def __init_subclass__(cls) -> None:
+            raise TypeError(f"type '{__name__}.ParamSpec' is not an acceptable base type")
+
+# 3.9
+else:
+
+    # Inherits from list as a workaround for Callable checks in Python < 3.9.2.
+    class ParamSpec(list, _DefaultMixin):
+        """Parameter specification variable.
+
+        Usage::
+
+           P = ParamSpec('P')
+
+        Parameter specification variables exist primarily for the benefit of static
+        type checkers.  They are used to forward the parameter types of one
+        callable to another callable, a pattern commonly found in higher order
+        functions and decorators.  They are only valid when used in ``Concatenate``,
+        or s the first argument to ``Callable``. In Python 3.10 and higher,
+        they are also supported in user-defined Generics at runtime.
+        See class Generic for more information on generic types.  An
+        example for annotating a decorator::
+
+           T = TypeVar('T')
+           P = ParamSpec('P')
+
+           def add_logging(f: Callable[P, T]) -> Callable[P, T]:
+               '''A type-safe decorator to add logging to a function.'''
+               def inner(*args: P.args, **kwargs: P.kwargs) -> T:
+                   logging.info(f'{f.__name__} was called')
+                   return f(*args, **kwargs)
+               return inner
+
+           @add_logging
+           def add_two(x: float, y: float) -> float:
+               '''Add two numbers together.'''
+               return x + y
+
+        Parameter specification variables defined with covariant=True or
+        contravariant=True can be used to declare covariant or contravariant
+        generic types.  These keyword arguments are valid, but their actual semantics
+        are yet to be decided.  See PEP 612 for details.
+
+        Parameter specification variables can be introspected. e.g.:
+
+           P.__name__ == 'T'
+           P.__bound__ == None
+           P.__covariant__ == False
+           P.__contravariant__ == False
+
+        Note that only parameter specification variables defined in global scope can
+        be pickled.
+        """
+
+        # Trick Generic __parameters__.
+        __class__ = typing.TypeVar
+
+        @property
+        def args(self):
+            return ParamSpecArgs(self)
+
+        @property
+        def kwargs(self):
+            return ParamSpecKwargs(self)
+
+        def __init__(self, name, *, bound=None, covariant=False, contravariant=False,
+                     infer_variance=False, default=NoDefault):
+            list.__init__(self, [self])
+            self.__name__ = name
+            self.__covariant__ = bool(covariant)
+            self.__contravariant__ = bool(contravariant)
+            self.__infer_variance__ = bool(infer_variance)
+            if bound:
+                self.__bound__ = typing._type_check(bound, 'Bound must be a type.')
+            else:
+                self.__bound__ = None
+            _DefaultMixin.__init__(self, default)
+
+            # for pickling:
+            def_mod = _caller()
+            if def_mod != 'typing_extensions':
+                self.__module__ = def_mod
+
+        def __repr__(self):
+            if self.__infer_variance__:
+                prefix = ''
+            elif self.__covariant__:
+                prefix = '+'
+            elif self.__contravariant__:
+                prefix = '-'
+            else:
+                prefix = '~'
+            return prefix + self.__name__
+
+        def __hash__(self):
+            return object.__hash__(self)
+
+        def __eq__(self, other):
+            return self is other
+
+        def __reduce__(self):
+            return self.__name__
+
+        # Hack to get typing._type_check to pass.
+        def __call__(self, *args, **kwargs):
+            pass
+
+
+# 3.9
+if not hasattr(typing, 'Concatenate'):
+    # Inherits from list as a workaround for Callable checks in Python < 3.9.2.
+
+    # 3.9.0-1
+    if not hasattr(typing, '_type_convert'):
+        def _type_convert(arg, module=None, *, allow_special_forms=False):
+            """For converting None to type(None), and strings to ForwardRef."""
+            if arg is None:
+                return type(None)
+            if isinstance(arg, str):
+                if sys.version_info <= (3, 9, 6):
+                    return ForwardRef(arg)
+                if sys.version_info <= (3, 9, 7):
+                    return ForwardRef(arg, module=module)
+                return ForwardRef(arg, module=module, is_class=allow_special_forms)
+            return arg
+    else:
+        _type_convert = typing._type_convert
+
+    class _ConcatenateGenericAlias(list):
+
+        # Trick Generic into looking into this for __parameters__.
+        __class__ = typing._GenericAlias
+
+        def __init__(self, origin, args):
+            super().__init__(args)
+            self.__origin__ = origin
+            self.__args__ = args
+
+        def __repr__(self):
+            _type_repr = typing._type_repr
+            return (f'{_type_repr(self.__origin__)}'
+                    f'[{", ".join(_type_repr(arg) for arg in self.__args__)}]')
+
+        def __hash__(self):
+            return hash((self.__origin__, self.__args__))
+
+        # Hack to get typing._type_check to pass in Generic.
+        def __call__(self, *args, **kwargs):
+            pass
+
+        @property
+        def __parameters__(self):
+            return tuple(
+                tp for tp in self.__args__ if isinstance(tp, (typing.TypeVar, ParamSpec))
+            )
+
+        # 3.9 used by __getitem__ below
+        def copy_with(self, params):
+            if isinstance(params[-1], _ConcatenateGenericAlias):
+                params = (*params[:-1], *params[-1].__args__)
+            elif isinstance(params[-1], (list, tuple)):
+                return (*params[:-1], *params[-1])
+            elif (not (params[-1] is ... or isinstance(params[-1], ParamSpec))):
+                raise TypeError("The last parameter to Concatenate should be a "
+                        "ParamSpec variable or ellipsis.")
+            return self.__class__(self.__origin__, params)
+
+        # 3.9; accessed during GenericAlias.__getitem__ when substituting
+        def __getitem__(self, args):
+            if self.__origin__ in (Generic, Protocol):
+                # Can't subscript Generic[...] or Protocol[...].
+                raise TypeError(f"Cannot subscript already-subscripted {self}")
+            if not self.__parameters__:
+                raise TypeError(f"{self} is not a generic class")
+
+            if not isinstance(args, tuple):
+                args = (args,)
+            args = _unpack_args(*(_type_convert(p) for p in args))
+            params = self.__parameters__
+            for param in params:
+                prepare = getattr(param, "__typing_prepare_subst__", None)
+                if prepare is not None:
+                    args = prepare(self, args)
+                # 3.9 & typing.ParamSpec
+                elif isinstance(param, ParamSpec):
+                    i = params.index(param)
+                    if (
+                        i == len(args)
+                        and getattr(param, '__default__', NoDefault) is not NoDefault
+                    ):
+                        args = [*args, param.__default__]
+                    if i >= len(args):
+                        raise TypeError(f"Too few arguments for {self}")
+                    # Special case for Z[[int, str, bool]] == Z[int, str, bool]
+                    if len(params) == 1 and not _is_param_expr(args[0]):
+                        assert i == 0
+                        args = (args,)
+                    elif (
+                        isinstance(args[i], list)
+                        # 3.9
+                        # This class inherits from list do not convert
+                        and not isinstance(args[i], _ConcatenateGenericAlias)
+                    ):
+                        args = (*args[:i], tuple(args[i]), *args[i + 1:])
+
+            alen = len(args)
+            plen = len(params)
+            if alen != plen:
+                raise TypeError(
+                    f"Too {'many' if alen > plen else 'few'} arguments for {self};"
+                    f" actual {alen}, expected {plen}"
+                )
+
+            subst = dict(zip(self.__parameters__, args))
+            # determine new args
+            new_args = []
+            for arg in self.__args__:
+                if isinstance(arg, type):
+                    new_args.append(arg)
+                    continue
+                if isinstance(arg, TypeVar):
+                    arg = subst[arg]
+                    if (
+                        (isinstance(arg, typing._GenericAlias) and _is_unpack(arg))
+                        or (
+                            hasattr(_types, "GenericAlias")
+                            and isinstance(arg, _types.GenericAlias)
+                            and getattr(arg, "__unpacked__", False)
+                        )
+                    ):
+                        raise TypeError(f"{arg} is not valid as type argument")
+
+                elif isinstance(arg,
+                    typing._GenericAlias
+                    if not hasattr(_types, "GenericAlias") else
+                    (typing._GenericAlias, _types.GenericAlias)
+                ):
+                    subparams = arg.__parameters__
+                    if subparams:
+                        subargs = tuple(subst[x] for x in subparams)
+                        arg = arg[subargs]
+                new_args.append(arg)
+            return self.copy_with(tuple(new_args))
+
+# 3.10+
+else:
+    _ConcatenateGenericAlias = typing._ConcatenateGenericAlias
+
+    # 3.10
+    if sys.version_info < (3, 11):
+
+        class _ConcatenateGenericAlias(typing._ConcatenateGenericAlias, _root=True):
+            # needed for checks in collections.abc.Callable to accept this class
+            __module__ = "typing"
+
+            def copy_with(self, params):
+                if isinstance(params[-1], (list, tuple)):
+                    return (*params[:-1], *params[-1])
+                if isinstance(params[-1], typing._ConcatenateGenericAlias):
+                    params = (*params[:-1], *params[-1].__args__)
+                elif not (params[-1] is ... or isinstance(params[-1], ParamSpec)):
+                    raise TypeError("The last parameter to Concatenate should be a "
+                            "ParamSpec variable or ellipsis.")
+                return super(typing._ConcatenateGenericAlias, self).copy_with(params)
+
+            def __getitem__(self, args):
+                value = super().__getitem__(args)
+                if isinstance(value, tuple) and any(_is_unpack(t) for t in value):
+                    return tuple(_unpack_args(*(n for n in value)))
+                return value
+
+
+# 3.9.2
+class _EllipsisDummy: ...
+
+
+# <=3.10
+def _create_concatenate_alias(origin, parameters):
+    if parameters[-1] is ... and sys.version_info < (3, 9, 2):
+        # Hack: Arguments must be types, replace it with one.
+        parameters = (*parameters[:-1], _EllipsisDummy)
+    if sys.version_info >= (3, 10, 3):
+        concatenate = _ConcatenateGenericAlias(origin, parameters,
+                                        _typevar_types=(TypeVar, ParamSpec),
+                                        _paramspec_tvars=True)
+    else:
+        concatenate = _ConcatenateGenericAlias(origin, parameters)
+    if parameters[-1] is not _EllipsisDummy:
+        return concatenate
+    # Remove dummy again
+    concatenate.__args__ = tuple(p if p is not _EllipsisDummy else ...
+                                    for p in concatenate.__args__)
+    if sys.version_info < (3, 10):
+        # backport needs __args__ adjustment only
+        return concatenate
+    concatenate.__parameters__ = tuple(p for p in concatenate.__parameters__
+                                        if p is not _EllipsisDummy)
+    return concatenate
+
+
+# <=3.10
+@typing._tp_cache
+def _concatenate_getitem(self, parameters):
+    if parameters == ():
+        raise TypeError("Cannot take a Concatenate of no types.")
+    if not isinstance(parameters, tuple):
+        parameters = (parameters,)
+    if not (parameters[-1] is ... or isinstance(parameters[-1], ParamSpec)):
+        raise TypeError("The last parameter to Concatenate should be a "
+                        "ParamSpec variable or ellipsis.")
+    msg = "Concatenate[arg, ...]: each arg must be a type."
+    parameters = (*(typing._type_check(p, msg) for p in parameters[:-1]),
+                    parameters[-1])
+    return _create_concatenate_alias(self, parameters)
+
+
+# 3.11+; Concatenate does not accept ellipsis in 3.10
+if sys.version_info >= (3, 11):
+    Concatenate = typing.Concatenate
+# <=3.10
+else:
+    @_ExtensionsSpecialForm
+    def Concatenate(self, parameters):
+        """Used in conjunction with ``ParamSpec`` and ``Callable`` to represent a
+        higher order function which adds, removes or transforms parameters of a
+        callable.
+
+        For example::
+
+           Callable[Concatenate[int, P], int]
+
+        See PEP 612 for detailed information.
+        """
+        return _concatenate_getitem(self, parameters)
+
+
+# 3.10+
+if hasattr(typing, 'TypeGuard'):
+    TypeGuard = typing.TypeGuard
+# 3.9
+else:
+    @_ExtensionsSpecialForm
+    def TypeGuard(self, parameters):
+        """Special typing form used to annotate the return type of a user-defined
+        type guard function.  ``TypeGuard`` only accepts a single type argument.
+        At runtime, functions marked this way should return a boolean.
+
+        ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static
+        type checkers to determine a more precise type of an expression within a
+        program's code flow.  Usually type narrowing is done by analyzing
+        conditional code flow and applying the narrowing to a block of code.  The
+        conditional expression here is sometimes referred to as a "type guard".
+
+        Sometimes it would be convenient to use a user-defined boolean function
+        as a type guard.  Such a function should use ``TypeGuard[...]`` as its
+        return type to alert static type checkers to this intention.
+
+        Using  ``-> TypeGuard`` tells the static type checker that for a given
+        function:
+
+        1. The return value is a boolean.
+        2. If the return value is ``True``, the type of its argument
+        is the type inside ``TypeGuard``.
+
+        For example::
+
+            def is_str(val: Union[str, float]):
+                # "isinstance" type guard
+                if isinstance(val, str):
+                    # Type of ``val`` is narrowed to ``str``
+                    ...
+                else:
+                    # Else, type of ``val`` is narrowed to ``float``.
+                    ...
+
+        Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower
+        form of ``TypeA`` (it can even be a wider form) and this may lead to
+        type-unsafe results.  The main reason is to allow for things like
+        narrowing ``List[object]`` to ``List[str]`` even though the latter is not
+        a subtype of the former, since ``List`` is invariant.  The responsibility of
+        writing type-safe type guards is left to the user.
+
+        ``TypeGuard`` also works with type variables.  For more information, see
+        PEP 647 (User-Defined Type Guards).
+        """
+        item = typing._type_check(parameters, f'{self} accepts only a single type.')
+        return typing._GenericAlias(self, (item,))
+
+
+# 3.13+
+if hasattr(typing, 'TypeIs'):
+    TypeIs = typing.TypeIs
+# <=3.12
+else:
+    @_ExtensionsSpecialForm
+    def TypeIs(self, parameters):
+        """Special typing form used to annotate the return type of a user-defined
+        type narrower function.  ``TypeIs`` only accepts a single type argument.
+        At runtime, functions marked this way should return a boolean.
+
+        ``TypeIs`` aims to benefit *type narrowing* -- a technique used by static
+        type checkers to determine a more precise type of an expression within a
+        program's code flow.  Usually type narrowing is done by analyzing
+        conditional code flow and applying the narrowing to a block of code.  The
+        conditional expression here is sometimes referred to as a "type guard".
+
+        Sometimes it would be convenient to use a user-defined boolean function
+        as a type guard.  Such a function should use ``TypeIs[...]`` as its
+        return type to alert static type checkers to this intention.
+
+        Using  ``-> TypeIs`` tells the static type checker that for a given
+        function:
+
+        1. The return value is a boolean.
+        2. If the return value is ``True``, the type of its argument
+        is the intersection of the type inside ``TypeIs`` and the argument's
+        previously known type.
+
+        For example::
+
+            def is_awaitable(val: object) -> TypeIs[Awaitable[Any]]:
+                return hasattr(val, '__await__')
+
+            def f(val: Union[int, Awaitable[int]]) -> int:
+                if is_awaitable(val):
+                    assert_type(val, Awaitable[int])
+                else:
+                    assert_type(val, int)
+
+        ``TypeIs`` also works with type variables.  For more information, see
+        PEP 742 (Narrowing types with TypeIs).
+        """
+        item = typing._type_check(parameters, f'{self} accepts only a single type.')
+        return typing._GenericAlias(self, (item,))
+
+
+# 3.14+?
+if hasattr(typing, 'TypeForm'):
+    TypeForm = typing.TypeForm
+# <=3.13
+else:
+    class _TypeFormForm(_ExtensionsSpecialForm, _root=True):
+        # TypeForm(X) is equivalent to X but indicates to the type checker
+        # that the object is a TypeForm.
+        def __call__(self, obj, /):
+            return obj
+
+    @_TypeFormForm
+    def TypeForm(self, parameters):
+        """A special form representing the value that results from the evaluation
+        of a type expression. This value encodes the information supplied in the
+        type expression, and it represents the type described by that type expression.
+
+        When used in a type expression, TypeForm describes a set of type form objects.
+        It accepts a single type argument, which must be a valid type expression.
+        ``TypeForm[T]`` describes the set of all type form objects that represent
+        the type T or types that are assignable to T.
+
+        Usage:
+
+            def cast[T](typ: TypeForm[T], value: Any) -> T: ...
+
+            reveal_type(cast(int, "x"))  # int
+
+        See PEP 747 for more information.
+        """
+        item = typing._type_check(parameters, f'{self} accepts only a single type.')
+        return typing._GenericAlias(self, (item,))
+
+
+
+
+if hasattr(typing, "LiteralString"):  # 3.11+
+    LiteralString = typing.LiteralString
+else:
+    @_SpecialForm
+    def LiteralString(self, params):
+        """Represents an arbitrary literal string.
+
+        Example::
+
+          from typing_extensions import LiteralString
+
+          def query(sql: LiteralString) -> ...:
+              ...
+
+          query("SELECT * FROM table")  # ok
+          query(f"SELECT * FROM {input()}")  # not ok
+
+        See PEP 675 for details.
+
+        """
+        raise TypeError(f"{self} is not subscriptable")
+
+
+if hasattr(typing, "Self"):  # 3.11+
+    Self = typing.Self
+else:
+    @_SpecialForm
+    def Self(self, params):
+        """Used to spell the type of "self" in classes.
+
+        Example::
+
+          from typing import Self
+
+          class ReturnsSelf:
+              def parse(self, data: bytes) -> Self:
+                  ...
+                  return self
+
+        """
+
+        raise TypeError(f"{self} is not subscriptable")
+
+
+if hasattr(typing, "Never"):  # 3.11+
+    Never = typing.Never
+else:
+    @_SpecialForm
+    def Never(self, params):
+        """The bottom type, a type that has no members.
+
+        This can be used to define a function that should never be
+        called, or a function that never returns::
+
+            from typing_extensions import Never
+
+            def never_call_me(arg: Never) -> None:
+                pass
+
+            def int_or_str(arg: int | str) -> None:
+                never_call_me(arg)  # type checker error
+                match arg:
+                    case int():
+                        print("It's an int")
+                    case str():
+                        print("It's a str")
+                    case _:
+                        never_call_me(arg)  # ok, arg is of type Never
+
+        """
+
+        raise TypeError(f"{self} is not subscriptable")
+
+
+if hasattr(typing, 'Required'):  # 3.11+
+    Required = typing.Required
+    NotRequired = typing.NotRequired
+else:  # <=3.10
+    @_ExtensionsSpecialForm
+    def Required(self, parameters):
+        """A special typing construct to mark a key of a total=False TypedDict
+        as required. For example:
+
+            class Movie(TypedDict, total=False):
+                title: Required[str]
+                year: int
+
+            m = Movie(
+                title='The Matrix',  # typechecker error if key is omitted
+                year=1999,
+            )
+
+        There is no runtime checking that a required key is actually provided
+        when instantiating a related TypedDict.
+        """
+        item = typing._type_check(parameters, f'{self._name} accepts only a single type.')
+        return typing._GenericAlias(self, (item,))
+
+    @_ExtensionsSpecialForm
+    def NotRequired(self, parameters):
+        """A special typing construct to mark a key of a TypedDict as
+        potentially missing. For example:
+
+            class Movie(TypedDict):
+                title: str
+                year: NotRequired[int]
+
+            m = Movie(
+                title='The Matrix',  # typechecker error if key is omitted
+                year=1999,
+            )
+        """
+        item = typing._type_check(parameters, f'{self._name} accepts only a single type.')
+        return typing._GenericAlias(self, (item,))
+
+
+if hasattr(typing, 'ReadOnly'):
+    ReadOnly = typing.ReadOnly
+else:  # <=3.12
+    @_ExtensionsSpecialForm
+    def ReadOnly(self, parameters):
+        """A special typing construct to mark an item of a TypedDict as read-only.
+
+        For example:
+
+            class Movie(TypedDict):
+                title: ReadOnly[str]
+                year: int
+
+            def mutate_movie(m: Movie) -> None:
+                m["year"] = 1992  # allowed
+                m["title"] = "The Matrix"  # typechecker error
+
+        There is no runtime checking for this property.
+        """
+        item = typing._type_check(parameters, f'{self._name} accepts only a single type.')
+        return typing._GenericAlias(self, (item,))
+
+
+_UNPACK_DOC = """\
+Type unpack operator.
+
+The type unpack operator takes the child types from some container type,
+such as `tuple[int, str]` or a `TypeVarTuple`, and 'pulls them out'. For
+example:
+
+  # For some generic class `Foo`:
+  Foo[Unpack[tuple[int, str]]]  # Equivalent to Foo[int, str]
+
+  Ts = TypeVarTuple('Ts')
+  # Specifies that `Bar` is generic in an arbitrary number of types.
+  # (Think of `Ts` as a tuple of an arbitrary number of individual
+  #  `TypeVar`s, which the `Unpack` is 'pulling out' directly into the
+  #  `Generic[]`.)
+  class Bar(Generic[Unpack[Ts]]): ...
+  Bar[int]  # Valid
+  Bar[int, str]  # Also valid
+
+From Python 3.11, this can also be done using the `*` operator:
+
+    Foo[*tuple[int, str]]
+    class Bar(Generic[*Ts]): ...
+
+The operator can also be used along with a `TypedDict` to annotate
+`**kwargs` in a function signature. For instance:
+
+  class Movie(TypedDict):
+    name: str
+    year: int
+
+  # This function expects two keyword arguments - *name* of type `str` and
+  # *year* of type `int`.
+  def foo(**kwargs: Unpack[Movie]): ...
+
+Note that there is only some runtime checking of this operator. Not
+everything the runtime allows may be accepted by static type checkers.
+
+For more information, see PEP 646 and PEP 692.
+"""
+
+
+if sys.version_info >= (3, 12):  # PEP 692 changed the repr of Unpack[]
+    Unpack = typing.Unpack
+
+    def _is_unpack(obj):
+        return get_origin(obj) is Unpack
+
+else:  # <=3.11
+    class _UnpackSpecialForm(_ExtensionsSpecialForm, _root=True):
+        def __init__(self, getitem):
+            super().__init__(getitem)
+            self.__doc__ = _UNPACK_DOC
+
+    class _UnpackAlias(typing._GenericAlias, _root=True):
+        if sys.version_info < (3, 11):
+            # needed for compatibility with Generic[Unpack[Ts]]
+            __class__ = typing.TypeVar
+
+        @property
+        def __typing_unpacked_tuple_args__(self):
+            assert self.__origin__ is Unpack
+            assert len(self.__args__) == 1
+            arg, = self.__args__
+            if isinstance(arg, (typing._GenericAlias, _types.GenericAlias)):
+                if arg.__origin__ is not tuple:
+                    raise TypeError("Unpack[...] must be used with a tuple type")
+                return arg.__args__
+            return None
+
+        @property
+        def __typing_is_unpacked_typevartuple__(self):
+            assert self.__origin__ is Unpack
+            assert len(self.__args__) == 1
+            return isinstance(self.__args__[0], TypeVarTuple)
+
+        def __getitem__(self, args):
+            if self.__typing_is_unpacked_typevartuple__:
+                return args
+            return super().__getitem__(args)
+
+    @_UnpackSpecialForm
+    def Unpack(self, parameters):
+        item = typing._type_check(parameters, f'{self._name} accepts only a single type.')
+        return _UnpackAlias(self, (item,))
+
+    def _is_unpack(obj):
+        return isinstance(obj, _UnpackAlias)
+
+
+def _unpack_args(*args):
+    newargs = []
+    for arg in args:
+        subargs = getattr(arg, '__typing_unpacked_tuple_args__', None)
+        if subargs is not None and (not (subargs and subargs[-1] is ...)):
+            newargs.extend(subargs)
+        else:
+            newargs.append(arg)
+    return newargs
+
+
+if _PEP_696_IMPLEMENTED:
+    from typing import TypeVarTuple
+
+elif hasattr(typing, "TypeVarTuple"):  # 3.11+
+
+    # Add default parameter - PEP 696
+    class TypeVarTuple(metaclass=_TypeVarLikeMeta):
+        """Type variable tuple."""
+
+        _backported_typevarlike = typing.TypeVarTuple
+
+        def __new__(cls, name, *, default=NoDefault):
+            tvt = typing.TypeVarTuple(name)
+            _set_default(tvt, default)
+            _set_module(tvt)
+
+            def _typevartuple_prepare_subst(alias, args):
+                params = alias.__parameters__
+                typevartuple_index = params.index(tvt)
+                for param in params[typevartuple_index + 1:]:
+                    if isinstance(param, TypeVarTuple):
+                        raise TypeError(
+                            f"More than one TypeVarTuple parameter in {alias}"
+                        )
+
+                alen = len(args)
+                plen = len(params)
+                left = typevartuple_index
+                right = plen - typevartuple_index - 1
+                var_tuple_index = None
+                fillarg = None
+                for k, arg in enumerate(args):
+                    if not isinstance(arg, type):
+                        subargs = getattr(arg, '__typing_unpacked_tuple_args__', None)
+                        if subargs and len(subargs) == 2 and subargs[-1] is ...:
+                            if var_tuple_index is not None:
+                                raise TypeError(
+                                    "More than one unpacked "
+                                    "arbitrary-length tuple argument"
+                                )
+                            var_tuple_index = k
+                            fillarg = subargs[0]
+                if var_tuple_index is not None:
+                    left = min(left, var_tuple_index)
+                    right = min(right, alen - var_tuple_index - 1)
+                elif left + right > alen:
+                    raise TypeError(f"Too few arguments for {alias};"
+                                    f" actual {alen}, expected at least {plen - 1}")
+                if left == alen - right and tvt.has_default():
+                    replacement = _unpack_args(tvt.__default__)
+                else:
+                    replacement = args[left: alen - right]
+
+                return (
+                    *args[:left],
+                    *([fillarg] * (typevartuple_index - left)),
+                    replacement,
+                    *([fillarg] * (plen - right - left - typevartuple_index - 1)),
+                    *args[alen - right:],
+                )
+
+            tvt.__typing_prepare_subst__ = _typevartuple_prepare_subst
+            return tvt
+
+        def __init_subclass__(self, *args, **kwds):
+            raise TypeError("Cannot subclass special typing classes")
+
+else:  # <=3.10
+    class TypeVarTuple(_DefaultMixin):
+        """Type variable tuple.
+
+        Usage::
+
+            Ts = TypeVarTuple('Ts')
+
+        In the same way that a normal type variable is a stand-in for a single
+        type such as ``int``, a type variable *tuple* is a stand-in for a *tuple*
+        type such as ``Tuple[int, str]``.
+
+        Type variable tuples can be used in ``Generic`` declarations.
+        Consider the following example::
+
+            class Array(Generic[*Ts]): ...
+
+        The ``Ts`` type variable tuple here behaves like ``tuple[T1, T2]``,
+        where ``T1`` and ``T2`` are type variables. To use these type variables
+        as type parameters of ``Array``, we must *unpack* the type variable tuple using
+        the star operator: ``*Ts``. The signature of ``Array`` then behaves
+        as if we had simply written ``class Array(Generic[T1, T2]): ...``.
+        In contrast to ``Generic[T1, T2]``, however, ``Generic[*Shape]`` allows
+        us to parameterise the class with an *arbitrary* number of type parameters.
+
+        Type variable tuples can be used anywhere a normal ``TypeVar`` can.
+        This includes class definitions, as shown above, as well as function
+        signatures and variable annotations::
+
+            class Array(Generic[*Ts]):
+
+                def __init__(self, shape: Tuple[*Ts]):
+                    self._shape: Tuple[*Ts] = shape
+
+                def get_shape(self) -> Tuple[*Ts]:
+                    return self._shape
+
+            shape = (Height(480), Width(640))
+            x: Array[Height, Width] = Array(shape)
+            y = abs(x)  # Inferred type is Array[Height, Width]
+            z = x + x   #        ...    is Array[Height, Width]
+            x.get_shape()  #     ...    is tuple[Height, Width]
+
+        """
+
+        # Trick Generic __parameters__.
+        __class__ = typing.TypeVar
+
+        def __iter__(self):
+            yield self.__unpacked__
+
+        def __init__(self, name, *, default=NoDefault):
+            self.__name__ = name
+            _DefaultMixin.__init__(self, default)
+
+            # for pickling:
+            def_mod = _caller()
+            if def_mod != 'typing_extensions':
+                self.__module__ = def_mod
+
+            self.__unpacked__ = Unpack[self]
+
+        def __repr__(self):
+            return self.__name__
+
+        def __hash__(self):
+            return object.__hash__(self)
+
+        def __eq__(self, other):
+            return self is other
+
+        def __reduce__(self):
+            return self.__name__
+
+        def __init_subclass__(self, *args, **kwds):
+            if '_root' not in kwds:
+                raise TypeError("Cannot subclass special typing classes")
+
+
+if hasattr(typing, "reveal_type"):  # 3.11+
+    reveal_type = typing.reveal_type
+else:  # <=3.10
+    def reveal_type(obj: T, /) -> T:
+        """Reveal the inferred type of a variable.
+
+        When a static type checker encounters a call to ``reveal_type()``,
+        it will emit the inferred type of the argument::
+
+            x: int = 1
+            reveal_type(x)
+
+        Running a static type checker (e.g., ``mypy``) on this example
+        will produce output similar to 'Revealed type is "builtins.int"'.
+
+        At runtime, the function prints the runtime type of the
+        argument and returns it unchanged.
+
+        """
+        print(f"Runtime type is {type(obj).__name__!r}", file=sys.stderr)
+        return obj
+
+
+if hasattr(typing, "_ASSERT_NEVER_REPR_MAX_LENGTH"):  # 3.11+
+    _ASSERT_NEVER_REPR_MAX_LENGTH = typing._ASSERT_NEVER_REPR_MAX_LENGTH
+else:  # <=3.10
+    _ASSERT_NEVER_REPR_MAX_LENGTH = 100
+
+
+if hasattr(typing, "assert_never"):  # 3.11+
+    assert_never = typing.assert_never
+else:  # <=3.10
+    def assert_never(arg: Never, /) -> Never:
+        """Assert to the type checker that a line of code is unreachable.
+
+        Example::
+
+            def int_or_str(arg: int | str) -> None:
+                match arg:
+                    case int():
+                        print("It's an int")
+                    case str():
+                        print("It's a str")
+                    case _:
+                        assert_never(arg)
+
+        If a type checker finds that a call to assert_never() is
+        reachable, it will emit an error.
+
+        At runtime, this throws an exception when called.
+
+        """
+        value = repr(arg)
+        if len(value) > _ASSERT_NEVER_REPR_MAX_LENGTH:
+            value = value[:_ASSERT_NEVER_REPR_MAX_LENGTH] + '...'
+        raise AssertionError(f"Expected code to be unreachable, but got: {value}")
+
+
+if sys.version_info >= (3, 12):  # 3.12+
+    # dataclass_transform exists in 3.11 but lacks the frozen_default parameter
+    dataclass_transform = typing.dataclass_transform
+else:  # <=3.11
+    def dataclass_transform(
+        *,
+        eq_default: bool = True,
+        order_default: bool = False,
+        kw_only_default: bool = False,
+        frozen_default: bool = False,
+        field_specifiers: typing.Tuple[
+            typing.Union[typing.Type[typing.Any], typing.Callable[..., typing.Any]],
+            ...
+        ] = (),
+        **kwargs: typing.Any,
+    ) -> typing.Callable[[T], T]:
+        """Decorator that marks a function, class, or metaclass as providing
+        dataclass-like behavior.
+
+        Example:
+
+            from typing_extensions import dataclass_transform
+
+            _T = TypeVar("_T")
+
+            # Used on a decorator function
+            @dataclass_transform()
+            def create_model(cls: type[_T]) -> type[_T]:
+                ...
+                return cls
+
+            @create_model
+            class CustomerModel:
+                id: int
+                name: str
+
+            # Used on a base class
+            @dataclass_transform()
+            class ModelBase: ...
+
+            class CustomerModel(ModelBase):
+                id: int
+                name: str
+
+            # Used on a metaclass
+            @dataclass_transform()
+            class ModelMeta(type): ...
+
+            class ModelBase(metaclass=ModelMeta): ...
+
+            class CustomerModel(ModelBase):
+                id: int
+                name: str
+
+        Each of the ``CustomerModel`` classes defined in this example will now
+        behave similarly to a dataclass created with the ``@dataclasses.dataclass``
+        decorator. For example, the type checker will synthesize an ``__init__``
+        method.
+
+        The arguments to this decorator can be used to customize this behavior:
+        - ``eq_default`` indicates whether the ``eq`` parameter is assumed to be
+          True or False if it is omitted by the caller.
+        - ``order_default`` indicates whether the ``order`` parameter is
+          assumed to be True or False if it is omitted by the caller.
+        - ``kw_only_default`` indicates whether the ``kw_only`` parameter is
+          assumed to be True or False if it is omitted by the caller.
+        - ``frozen_default`` indicates whether the ``frozen`` parameter is
+          assumed to be True or False if it is omitted by the caller.
+        - ``field_specifiers`` specifies a static list of supported classes
+          or functions that describe fields, similar to ``dataclasses.field()``.
+
+        At runtime, this decorator records its arguments in the
+        ``__dataclass_transform__`` attribute on the decorated object.
+
+        See PEP 681 for details.
+
+        """
+        def decorator(cls_or_fn):
+            cls_or_fn.__dataclass_transform__ = {
+                "eq_default": eq_default,
+                "order_default": order_default,
+                "kw_only_default": kw_only_default,
+                "frozen_default": frozen_default,
+                "field_specifiers": field_specifiers,
+                "kwargs": kwargs,
+            }
+            return cls_or_fn
+        return decorator
+
+
+if hasattr(typing, "override"):  # 3.12+
+    override = typing.override
+else:  # <=3.11
+    _F = typing.TypeVar("_F", bound=typing.Callable[..., typing.Any])
+
+    def override(arg: _F, /) -> _F:
+        """Indicate that a method is intended to override a method in a base class.
+
+        Usage:
+
+            class Base:
+                def method(self) -> None:
+                    pass
+
+            class Child(Base):
+                @override
+                def method(self) -> None:
+                    super().method()
+
+        When this decorator is applied to a method, the type checker will
+        validate that it overrides a method with the same name on a base class.
+        This helps prevent bugs that may occur when a base class is changed
+        without an equivalent change to a child class.
+
+        There is no runtime checking of these properties. The decorator
+        sets the ``__override__`` attribute to ``True`` on the decorated object
+        to allow runtime introspection.
+
+        See PEP 698 for details.
+
+        """
+        try:
+            arg.__override__ = True
+        except (AttributeError, TypeError):
+            # Skip the attribute silently if it is not writable.
+            # AttributeError happens if the object has __slots__ or a
+            # read-only property, TypeError if it's a builtin class.
+            pass
+        return arg
+
+
+# Python 3.13.3+ contains a fix for the wrapped __new__
+if sys.version_info >= (3, 13, 3):
+    deprecated = warnings.deprecated
+else:
+    _T = typing.TypeVar("_T")
+
+    class deprecated:
+        """Indicate that a class, function or overload is deprecated.
+
+        When this decorator is applied to an object, the type checker
+        will generate a diagnostic on usage of the deprecated object.
+
+        Usage:
+
+            @deprecated("Use B instead")
+            class A:
+                pass
+
+            @deprecated("Use g instead")
+            def f():
+                pass
+
+            @overload
+            @deprecated("int support is deprecated")
+            def g(x: int) -> int: ...
+            @overload
+            def g(x: str) -> int: ...
+
+        The warning specified by *category* will be emitted at runtime
+        on use of deprecated objects. For functions, that happens on calls;
+        for classes, on instantiation and on creation of subclasses.
+        If the *category* is ``None``, no warning is emitted at runtime.
+        The *stacklevel* determines where the
+        warning is emitted. If it is ``1`` (the default), the warning
+        is emitted at the direct caller of the deprecated object; if it
+        is higher, it is emitted further up the stack.
+        Static type checker behavior is not affected by the *category*
+        and *stacklevel* arguments.
+
+        The deprecation message passed to the decorator is saved in the
+        ``__deprecated__`` attribute on the decorated object.
+        If applied to an overload, the decorator
+        must be after the ``@overload`` decorator for the attribute to
+        exist on the overload as returned by ``get_overloads()``.
+
+        See PEP 702 for details.
+
+        """
+        def __init__(
+            self,
+            message: str,
+            /,
+            *,
+            category: typing.Optional[typing.Type[Warning]] = DeprecationWarning,
+            stacklevel: int = 1,
+        ) -> None:
+            if not isinstance(message, str):
+                raise TypeError(
+                    "Expected an object of type str for 'message', not "
+                    f"{type(message).__name__!r}"
+                )
+            self.message = message
+            self.category = category
+            self.stacklevel = stacklevel
+
+        def __call__(self, arg: _T, /) -> _T:
+            # Make sure the inner functions created below don't
+            # retain a reference to self.
+            msg = self.message
+            category = self.category
+            stacklevel = self.stacklevel
+            if category is None:
+                arg.__deprecated__ = msg
+                return arg
+            elif isinstance(arg, type):
+                import functools
+                from types import MethodType
+
+                original_new = arg.__new__
+
+                @functools.wraps(original_new)
+                def __new__(cls, /, *args, **kwargs):
+                    if cls is arg:
+                        warnings.warn(msg, category=category, stacklevel=stacklevel + 1)
+                    if original_new is not object.__new__:
+                        return original_new(cls, *args, **kwargs)
+                    # Mirrors a similar check in object.__new__.
+                    elif cls.__init__ is object.__init__ and (args or kwargs):
+                        raise TypeError(f"{cls.__name__}() takes no arguments")
+                    else:
+                        return original_new(cls)
+
+                arg.__new__ = staticmethod(__new__)
+
+                original_init_subclass = arg.__init_subclass__
+                # We need slightly different behavior if __init_subclass__
+                # is a bound method (likely if it was implemented in Python)
+                if isinstance(original_init_subclass, MethodType):
+                    original_init_subclass = original_init_subclass.__func__
+
+                    @functools.wraps(original_init_subclass)
+                    def __init_subclass__(*args, **kwargs):
+                        warnings.warn(msg, category=category, stacklevel=stacklevel + 1)
+                        return original_init_subclass(*args, **kwargs)
+
+                    arg.__init_subclass__ = classmethod(__init_subclass__)
+                # Or otherwise, which likely means it's a builtin such as
+                # object's implementation of __init_subclass__.
+                else:
+                    @functools.wraps(original_init_subclass)
+                    def __init_subclass__(*args, **kwargs):
+                        warnings.warn(msg, category=category, stacklevel=stacklevel + 1)
+                        return original_init_subclass(*args, **kwargs)
+
+                    arg.__init_subclass__ = __init_subclass__
+
+                arg.__deprecated__ = __new__.__deprecated__ = msg
+                __init_subclass__.__deprecated__ = msg
+                return arg
+            elif callable(arg):
+                import asyncio.coroutines
+                import functools
+                import inspect
+
+                @functools.wraps(arg)
+                def wrapper(*args, **kwargs):
+                    warnings.warn(msg, category=category, stacklevel=stacklevel + 1)
+                    return arg(*args, **kwargs)
+
+                if asyncio.coroutines.iscoroutinefunction(arg):
+                    if sys.version_info >= (3, 12):
+                        wrapper = inspect.markcoroutinefunction(wrapper)
+                    else:
+                        wrapper._is_coroutine = asyncio.coroutines._is_coroutine
+
+                arg.__deprecated__ = wrapper.__deprecated__ = msg
+                return wrapper
+            else:
+                raise TypeError(
+                    "@deprecated decorator with non-None category must be applied to "
+                    f"a class or callable, not {arg!r}"
+                )
+
+if sys.version_info < (3, 10):
+    def _is_param_expr(arg):
+        return arg is ... or isinstance(
+            arg, (tuple, list, ParamSpec, _ConcatenateGenericAlias)
+        )
+else:
+    def _is_param_expr(arg):
+        return arg is ... or isinstance(
+            arg,
+            (
+                tuple,
+                list,
+                ParamSpec,
+                _ConcatenateGenericAlias,
+                typing._ConcatenateGenericAlias,
+            ),
+        )
+
+
+# We have to do some monkey patching to deal with the dual nature of
+# Unpack/TypeVarTuple:
+# - We want Unpack to be a kind of TypeVar so it gets accepted in
+#   Generic[Unpack[Ts]]
+# - We want it to *not* be treated as a TypeVar for the purposes of
+#   counting generic parameters, so that when we subscript a generic,
+#   the runtime doesn't try to substitute the Unpack with the subscripted type.
+if not hasattr(typing, "TypeVarTuple"):
+    def _check_generic(cls, parameters, elen=_marker):
+        """Check correct count for parameters of a generic cls (internal helper).
+
+        This gives a nice error message in case of count mismatch.
+        """
+        # If substituting a single ParamSpec with multiple arguments
+        # we do not check the count
+        if (inspect.isclass(cls) and issubclass(cls, typing.Generic)
+            and len(cls.__parameters__) == 1
+            and isinstance(cls.__parameters__[0], ParamSpec)
+            and parameters
+            and not _is_param_expr(parameters[0])
+        ):
+            # Generic modifies parameters variable, but here we cannot do this
+            return
+
+        if not elen:
+            raise TypeError(f"{cls} is not a generic class")
+        if elen is _marker:
+            if not hasattr(cls, "__parameters__") or not cls.__parameters__:
+                raise TypeError(f"{cls} is not a generic class")
+            elen = len(cls.__parameters__)
+        alen = len(parameters)
+        if alen != elen:
+            expect_val = elen
+            if hasattr(cls, "__parameters__"):
+                parameters = [p for p in cls.__parameters__ if not _is_unpack(p)]
+                num_tv_tuples = sum(isinstance(p, TypeVarTuple) for p in parameters)
+                if (num_tv_tuples > 0) and (alen >= elen - num_tv_tuples):
+                    return
+
+                # deal with TypeVarLike defaults
+                # required TypeVarLikes cannot appear after a defaulted one.
+                if alen < elen:
+                    # since we validate TypeVarLike default in _collect_type_vars
+                    # or _collect_parameters we can safely check parameters[alen]
+                    if (
+                        getattr(parameters[alen], '__default__', NoDefault)
+                        is not NoDefault
+                    ):
+                        return
+
+                    num_default_tv = sum(getattr(p, '__default__', NoDefault)
+                                         is not NoDefault for p in parameters)
+
+                    elen -= num_default_tv
+
+                    expect_val = f"at least {elen}"
+
+            things = "arguments" if sys.version_info >= (3, 10) else "parameters"
+            raise TypeError(f"Too {'many' if alen > elen else 'few'} {things}"
+                            f" for {cls}; actual {alen}, expected {expect_val}")
+else:
+    # Python 3.11+
+
+    def _check_generic(cls, parameters, elen):
+        """Check correct count for parameters of a generic cls (internal helper).
+
+        This gives a nice error message in case of count mismatch.
+        """
+        if not elen:
+            raise TypeError(f"{cls} is not a generic class")
+        alen = len(parameters)
+        if alen != elen:
+            expect_val = elen
+            if hasattr(cls, "__parameters__"):
+                parameters = [p for p in cls.__parameters__ if not _is_unpack(p)]
+
+                # deal with TypeVarLike defaults
+                # required TypeVarLikes cannot appear after a defaulted one.
+                if alen < elen:
+                    # since we validate TypeVarLike default in _collect_type_vars
+                    # or _collect_parameters we can safely check parameters[alen]
+                    if (
+                        getattr(parameters[alen], '__default__', NoDefault)
+                        is not NoDefault
+                    ):
+                        return
+
+                    num_default_tv = sum(getattr(p, '__default__', NoDefault)
+                                         is not NoDefault for p in parameters)
+
+                    elen -= num_default_tv
+
+                    expect_val = f"at least {elen}"
+
+            raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments"
+                            f" for {cls}; actual {alen}, expected {expect_val}")
+
+if not _PEP_696_IMPLEMENTED:
+    typing._check_generic = _check_generic
+
+
+def _has_generic_or_protocol_as_origin() -> bool:
+    try:
+        frame = sys._getframe(2)
+    # - Catch AttributeError: not all Python implementations have sys._getframe()
+    # - Catch ValueError: maybe we're called from an unexpected module
+    #   and the call stack isn't deep enough
+    except (AttributeError, ValueError):
+        return False  # err on the side of leniency
+    else:
+        # If we somehow get invoked from outside typing.py,
+        # also err on the side of leniency
+        if frame.f_globals.get("__name__") != "typing":
+            return False
+        origin = frame.f_locals.get("origin")
+        # Cannot use "in" because origin may be an object with a buggy __eq__ that
+        # throws an error.
+        return origin is typing.Generic or origin is Protocol or origin is typing.Protocol
+
+
+_TYPEVARTUPLE_TYPES = {TypeVarTuple, getattr(typing, "TypeVarTuple", None)}
+
+
+def _is_unpacked_typevartuple(x) -> bool:
+    if get_origin(x) is not Unpack:
+        return False
+    args = get_args(x)
+    return (
+        bool(args)
+        and len(args) == 1
+        and type(args[0]) in _TYPEVARTUPLE_TYPES
+    )
+
+
+# Python 3.11+ _collect_type_vars was renamed to _collect_parameters
+if hasattr(typing, '_collect_type_vars'):
+    def _collect_type_vars(types, typevar_types=None):
+        """Collect all type variable contained in types in order of
+        first appearance (lexicographic order). For example::
+
+            _collect_type_vars((T, List[S, T])) == (T, S)
+        """
+        if typevar_types is None:
+            typevar_types = typing.TypeVar
+        tvars = []
+
+        # A required TypeVarLike cannot appear after a TypeVarLike with a default
+        # if it was a direct call to `Generic[]` or `Protocol[]`
+        enforce_default_ordering = _has_generic_or_protocol_as_origin()
+        default_encountered = False
+
+        # Also, a TypeVarLike with a default cannot appear after a TypeVarTuple
+        type_var_tuple_encountered = False
+
+        for t in types:
+            if _is_unpacked_typevartuple(t):
+                type_var_tuple_encountered = True
+            elif (
+                isinstance(t, typevar_types) and not isinstance(t, _UnpackAlias)
+                and t not in tvars
+            ):
+                if enforce_default_ordering:
+                    has_default = getattr(t, '__default__', NoDefault) is not NoDefault
+                    if has_default:
+                        if type_var_tuple_encountered:
+                            raise TypeError('Type parameter with a default'
+                                            ' follows TypeVarTuple')
+                        default_encountered = True
+                    elif default_encountered:
+                        raise TypeError(f'Type parameter {t!r} without a default'
+                                        ' follows type parameter with a default')
+
+                tvars.append(t)
+            if _should_collect_from_parameters(t):
+                tvars.extend([t for t in t.__parameters__ if t not in tvars])
+            elif isinstance(t, tuple):
+                # Collect nested type_vars
+                # tuple wrapped by  _prepare_paramspec_params(cls, params)
+                for x in t:
+                    for collected in _collect_type_vars([x]):
+                        if collected not in tvars:
+                            tvars.append(collected)
+        return tuple(tvars)
+
+    typing._collect_type_vars = _collect_type_vars
+else:
+    def _collect_parameters(args):
+        """Collect all type variables and parameter specifications in args
+        in order of first appearance (lexicographic order).
+
+        For example::
+
+            assert _collect_parameters((T, Callable[P, T])) == (T, P)
+        """
+        parameters = []
+
+        # A required TypeVarLike cannot appear after a TypeVarLike with default
+        # if it was a direct call to `Generic[]` or `Protocol[]`
+        enforce_default_ordering = _has_generic_or_protocol_as_origin()
+        default_encountered = False
+
+        # Also, a TypeVarLike with a default cannot appear after a TypeVarTuple
+        type_var_tuple_encountered = False
+
+        for t in args:
+            if isinstance(t, type):
+                # We don't want __parameters__ descriptor of a bare Python class.
+                pass
+            elif isinstance(t, tuple):
+                # `t` might be a tuple, when `ParamSpec` is substituted with
+                # `[T, int]`, or `[int, *Ts]`, etc.
+                for x in t:
+                    for collected in _collect_parameters([x]):
+                        if collected not in parameters:
+                            parameters.append(collected)
+            elif hasattr(t, '__typing_subst__'):
+                if t not in parameters:
+                    if enforce_default_ordering:
+                        has_default = (
+                            getattr(t, '__default__', NoDefault) is not NoDefault
+                        )
+
+                        if type_var_tuple_encountered and has_default:
+                            raise TypeError('Type parameter with a default'
+                                            ' follows TypeVarTuple')
+
+                        if has_default:
+                            default_encountered = True
+                        elif default_encountered:
+                            raise TypeError(f'Type parameter {t!r} without a default'
+                                            ' follows type parameter with a default')
+
+                    parameters.append(t)
+            else:
+                if _is_unpacked_typevartuple(t):
+                    type_var_tuple_encountered = True
+                for x in getattr(t, '__parameters__', ()):
+                    if x not in parameters:
+                        parameters.append(x)
+
+        return tuple(parameters)
+
+    if not _PEP_696_IMPLEMENTED:
+        typing._collect_parameters = _collect_parameters
+
+# Backport typing.NamedTuple as it exists in Python 3.13.
+# In 3.11, the ability to define generic `NamedTuple`s was supported.
+# This was explicitly disallowed in 3.9-3.10, and only half-worked in <=3.8.
+# On 3.12, we added __orig_bases__ to call-based NamedTuples
+# On 3.13, we deprecated kwargs-based NamedTuples
+if sys.version_info >= (3, 13):
+    NamedTuple = typing.NamedTuple
+else:
+    def _make_nmtuple(name, types, module, defaults=()):
+        fields = [n for n, t in types]
+        annotations = {n: typing._type_check(t, f"field {n} annotation must be a type")
+                       for n, t in types}
+        nm_tpl = collections.namedtuple(name, fields,
+                                        defaults=defaults, module=module)
+        nm_tpl.__annotations__ = nm_tpl.__new__.__annotations__ = annotations
+        return nm_tpl
+
+    _prohibited_namedtuple_fields = typing._prohibited
+    _special_namedtuple_fields = frozenset({'__module__', '__name__', '__annotations__'})
+
+    class _NamedTupleMeta(type):
+        def __new__(cls, typename, bases, ns):
+            assert _NamedTuple in bases
+            for base in bases:
+                if base is not _NamedTuple and base is not typing.Generic:
+                    raise TypeError(
+                        'can only inherit from a NamedTuple type and Generic')
+            bases = tuple(tuple if base is _NamedTuple else base for base in bases)
+            if "__annotations__" in ns:
+                types = ns["__annotations__"]
+            elif "__annotate__" in ns:
+                # TODO: Use inspect.VALUE here, and make the annotations lazily evaluated
+                types = ns["__annotate__"](1)
+            else:
+                types = {}
+            default_names = []
+            for field_name in types:
+                if field_name in ns:
+                    default_names.append(field_name)
+                elif default_names:
+                    raise TypeError(f"Non-default namedtuple field {field_name} "
+                                    f"cannot follow default field"
+                                    f"{'s' if len(default_names) > 1 else ''} "
+                                    f"{', '.join(default_names)}")
+            nm_tpl = _make_nmtuple(
+                typename, types.items(),
+                defaults=[ns[n] for n in default_names],
+                module=ns['__module__']
+            )
+            nm_tpl.__bases__ = bases
+            if typing.Generic in bases:
+                if hasattr(typing, '_generic_class_getitem'):  # 3.12+
+                    nm_tpl.__class_getitem__ = classmethod(typing._generic_class_getitem)
+                else:
+                    class_getitem = typing.Generic.__class_getitem__.__func__
+                    nm_tpl.__class_getitem__ = classmethod(class_getitem)
+            # update from user namespace without overriding special namedtuple attributes
+            for key, val in ns.items():
+                if key in _prohibited_namedtuple_fields:
+                    raise AttributeError("Cannot overwrite NamedTuple attribute " + key)
+                elif key not in _special_namedtuple_fields:
+                    if key not in nm_tpl._fields:
+                        setattr(nm_tpl, key, ns[key])
+                    try:
+                        set_name = type(val).__set_name__
+                    except AttributeError:
+                        pass
+                    else:
+                        try:
+                            set_name(val, nm_tpl, key)
+                        except BaseException as e:
+                            msg = (
+                                f"Error calling __set_name__ on {type(val).__name__!r} "
+                                f"instance {key!r} in {typename!r}"
+                            )
+                            # BaseException.add_note() existed on py311,
+                            # but the __set_name__ machinery didn't start
+                            # using add_note() until py312.
+                            # Making sure exceptions are raised in the same way
+                            # as in "normal" classes seems most important here.
+                            if sys.version_info >= (3, 12):
+                                e.add_note(msg)
+                                raise
+                            else:
+                                raise RuntimeError(msg) from e
+
+            if typing.Generic in bases:
+                nm_tpl.__init_subclass__()
+            return nm_tpl
+
+    _NamedTuple = type.__new__(_NamedTupleMeta, 'NamedTuple', (), {})
+
+    def _namedtuple_mro_entries(bases):
+        assert NamedTuple in bases
+        return (_NamedTuple,)
+
+    def NamedTuple(typename, fields=_marker, /, **kwargs):
+        """Typed version of namedtuple.
+
+        Usage::
+
+            class Employee(NamedTuple):
+                name: str
+                id: int
+
+        This is equivalent to::
+
+            Employee = collections.namedtuple('Employee', ['name', 'id'])
+
+        The resulting class has an extra __annotations__ attribute, giving a
+        dict that maps field names to types.  (The field names are also in
+        the _fields attribute, which is part of the namedtuple API.)
+        An alternative equivalent functional syntax is also accepted::
+
+            Employee = NamedTuple('Employee', [('name', str), ('id', int)])
+        """
+        if fields is _marker:
+            if kwargs:
+                deprecated_thing = "Creating NamedTuple classes using keyword arguments"
+                deprecation_msg = (
+                    "{name} is deprecated and will be disallowed in Python {remove}. "
+                    "Use the class-based or functional syntax instead."
+                )
+            else:
+                deprecated_thing = "Failing to pass a value for the 'fields' parameter"
+                example = f"`{typename} = NamedTuple({typename!r}, [])`"
+                deprecation_msg = (
+                    "{name} is deprecated and will be disallowed in Python {remove}. "
+                    "To create a NamedTuple class with 0 fields "
+                    "using the functional syntax, "
+                    "pass an empty list, e.g. "
+                ) + example + "."
+        elif fields is None:
+            if kwargs:
+                raise TypeError(
+                    "Cannot pass `None` as the 'fields' parameter "
+                    "and also specify fields using keyword arguments"
+                )
+            else:
+                deprecated_thing = "Passing `None` as the 'fields' parameter"
+                example = f"`{typename} = NamedTuple({typename!r}, [])`"
+                deprecation_msg = (
+                    "{name} is deprecated and will be disallowed in Python {remove}. "
+                    "To create a NamedTuple class with 0 fields "
+                    "using the functional syntax, "
+                    "pass an empty list, e.g. "
+                ) + example + "."
+        elif kwargs:
+            raise TypeError("Either list of fields or keywords"
+                            " can be provided to NamedTuple, not both")
+        if fields is _marker or fields is None:
+            warnings.warn(
+                deprecation_msg.format(name=deprecated_thing, remove="3.15"),
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            fields = kwargs.items()
+        nt = _make_nmtuple(typename, fields, module=_caller())
+        nt.__orig_bases__ = (NamedTuple,)
+        return nt
+
+    NamedTuple.__mro_entries__ = _namedtuple_mro_entries
+
+
+if hasattr(collections.abc, "Buffer"):
+    Buffer = collections.abc.Buffer
+else:
+    class Buffer(abc.ABC):  # noqa: B024
+        """Base class for classes that implement the buffer protocol.
+
+        The buffer protocol allows Python objects to expose a low-level
+        memory buffer interface. Before Python 3.12, it is not possible
+        to implement the buffer protocol in pure Python code, or even
+        to check whether a class implements the buffer protocol. In
+        Python 3.12 and higher, the ``__buffer__`` method allows access
+        to the buffer protocol from Python code, and the
+        ``collections.abc.Buffer`` ABC allows checking whether a class
+        implements the buffer protocol.
+
+        To indicate support for the buffer protocol in earlier versions,
+        inherit from this ABC, either in a stub file or at runtime,
+        or use ABC registration. This ABC provides no methods, because
+        there is no Python-accessible methods shared by pre-3.12 buffer
+        classes. It is useful primarily for static checks.
+
+        """
+
+    # As a courtesy, register the most common stdlib buffer classes.
+    Buffer.register(memoryview)
+    Buffer.register(bytearray)
+    Buffer.register(bytes)
+
+
+# Backport of types.get_original_bases, available on 3.12+ in CPython
+if hasattr(_types, "get_original_bases"):
+    get_original_bases = _types.get_original_bases
+else:
+    def get_original_bases(cls, /):
+        """Return the class's "original" bases prior to modification by `__mro_entries__`.
+
+        Examples::
+
+            from typing import TypeVar, Generic
+            from typing_extensions import NamedTuple, TypedDict
+
+            T = TypeVar("T")
+            class Foo(Generic[T]): ...
+            class Bar(Foo[int], float): ...
+            class Baz(list[str]): ...
+            Eggs = NamedTuple("Eggs", [("a", int), ("b", str)])
+            Spam = TypedDict("Spam", {"a": int, "b": str})
+
+            assert get_original_bases(Bar) == (Foo[int], float)
+            assert get_original_bases(Baz) == (list[str],)
+            assert get_original_bases(Eggs) == (NamedTuple,)
+            assert get_original_bases(Spam) == (TypedDict,)
+            assert get_original_bases(int) == (object,)
+        """
+        try:
+            return cls.__dict__.get("__orig_bases__", cls.__bases__)
+        except AttributeError:
+            raise TypeError(
+                f'Expected an instance of type, not {type(cls).__name__!r}'
+            ) from None
+
+
+# NewType is a class on Python 3.10+, making it pickleable
+# The error message for subclassing instances of NewType was improved on 3.11+
+if sys.version_info >= (3, 11):
+    NewType = typing.NewType
+else:
+    class NewType:
+        """NewType creates simple unique types with almost zero
+        runtime overhead. NewType(name, tp) is considered a subtype of tp
+        by static type checkers. At runtime, NewType(name, tp) returns
+        a dummy callable that simply returns its argument. Usage::
+            UserId = NewType('UserId', int)
+            def name_by_id(user_id: UserId) -> str:
+                ...
+            UserId('user')          # Fails type check
+            name_by_id(42)          # Fails type check
+            name_by_id(UserId(42))  # OK
+            num = UserId(5) + 1     # type: int
+        """
+
+        def __call__(self, obj, /):
+            return obj
+
+        def __init__(self, name, tp):
+            self.__qualname__ = name
+            if '.' in name:
+                name = name.rpartition('.')[-1]
+            self.__name__ = name
+            self.__supertype__ = tp
+            def_mod = _caller()
+            if def_mod != 'typing_extensions':
+                self.__module__ = def_mod
+
+        def __mro_entries__(self, bases):
+            # We defined __mro_entries__ to get a better error message
+            # if a user attempts to subclass a NewType instance. bpo-46170
+            supercls_name = self.__name__
+
+            class Dummy:
+                def __init_subclass__(cls):
+                    subcls_name = cls.__name__
+                    raise TypeError(
+                        f"Cannot subclass an instance of NewType. "
+                        f"Perhaps you were looking for: "
+                        f"`{subcls_name} = NewType({subcls_name!r}, {supercls_name})`"
+                    )
+
+            return (Dummy,)
+
+        def __repr__(self):
+            return f'{self.__module__}.{self.__qualname__}'
+
+        def __reduce__(self):
+            return self.__qualname__
+
+        if sys.version_info >= (3, 10):
+            # PEP 604 methods
+            # It doesn't make sense to have these methods on Python <3.10
+
+            def __or__(self, other):
+                return typing.Union[self, other]
+
+            def __ror__(self, other):
+                return typing.Union[other, self]
+
+
+if sys.version_info >= (3, 14):
+    TypeAliasType = typing.TypeAliasType
+# <=3.13
+else:
+    if sys.version_info >= (3, 12):
+        # 3.12-3.13
+        def _is_unionable(obj):
+            """Corresponds to is_unionable() in unionobject.c in CPython."""
+            return obj is None or isinstance(obj, (
+                type,
+                _types.GenericAlias,
+                _types.UnionType,
+                typing.TypeAliasType,
+                TypeAliasType,
+            ))
+    else:
+        # <=3.11
+        def _is_unionable(obj):
+            """Corresponds to is_unionable() in unionobject.c in CPython."""
+            return obj is None or isinstance(obj, (
+                type,
+                _types.GenericAlias,
+                _types.UnionType,
+                TypeAliasType,
+            ))
+
+    if sys.version_info < (3, 10):
+        # Copied and pasted from https://github.com/python/cpython/blob/986a4e1b6fcae7fe7a1d0a26aea446107dd58dd2/Objects/genericaliasobject.c#L568-L582,
+        # so that we emulate the behaviour of `types.GenericAlias`
+        # on the latest versions of CPython
+        _ATTRIBUTE_DELEGATION_EXCLUSIONS = frozenset({
+            "__class__",
+            "__bases__",
+            "__origin__",
+            "__args__",
+            "__unpacked__",
+            "__parameters__",
+            "__typing_unpacked_tuple_args__",
+            "__mro_entries__",
+            "__reduce_ex__",
+            "__reduce__",
+            "__copy__",
+            "__deepcopy__",
+        })
+
+        class _TypeAliasGenericAlias(typing._GenericAlias, _root=True):
+            def __getattr__(self, attr):
+                if attr in _ATTRIBUTE_DELEGATION_EXCLUSIONS:
+                    return object.__getattr__(self, attr)
+                return getattr(self.__origin__, attr)
+
+
+    class TypeAliasType:
+        """Create named, parameterized type aliases.
+
+        This provides a backport of the new `type` statement in Python 3.12:
+
+            type ListOrSet[T] = list[T] | set[T]
+
+        is equivalent to:
+
+            T = TypeVar("T")
+            ListOrSet = TypeAliasType("ListOrSet", list[T] | set[T], type_params=(T,))
+
+        The name ListOrSet can then be used as an alias for the type it refers to.
+
+        The type_params argument should contain all the type parameters used
+        in the value of the type alias. If the alias is not generic, this
+        argument is omitted.
+
+        Static type checkers should only support type aliases declared using
+        TypeAliasType that follow these rules:
+
+        - The first argument (the name) must be a string literal.
+        - The TypeAliasType instance must be immediately assigned to a variable
+          of the same name. (For example, 'X = TypeAliasType("Y", int)' is invalid,
+          as is 'X, Y = TypeAliasType("X", int), TypeAliasType("Y", int)').
+
+        """
+
+        def __init__(self, name: str, value, *, type_params=()):
+            if not isinstance(name, str):
+                raise TypeError("TypeAliasType name must be a string")
+            if not isinstance(type_params, tuple):
+                raise TypeError("type_params must be a tuple")
+            self.__value__ = value
+            self.__type_params__ = type_params
+
+            default_value_encountered = False
+            parameters = []
+            for type_param in type_params:
+                if (
+                    not isinstance(type_param, (TypeVar, TypeVarTuple, ParamSpec))
+                    # <=3.11
+                    # Unpack Backport passes isinstance(type_param, TypeVar)
+                    or _is_unpack(type_param)
+                ):
+                    raise TypeError(f"Expected a type param, got {type_param!r}")
+                has_default = (
+                    getattr(type_param, '__default__', NoDefault) is not NoDefault
+                )
+                if default_value_encountered and not has_default:
+                    raise TypeError(f"non-default type parameter '{type_param!r}'"
+                                    " follows default type parameter")
+                if has_default:
+                    default_value_encountered = True
+                if isinstance(type_param, TypeVarTuple):
+                    parameters.extend(type_param)
+                else:
+                    parameters.append(type_param)
+            self.__parameters__ = tuple(parameters)
+            def_mod = _caller()
+            if def_mod != 'typing_extensions':
+                self.__module__ = def_mod
+            # Setting this attribute closes the TypeAliasType from further modification
+            self.__name__ = name
+
+        def __setattr__(self, name: str, value: object, /) -> None:
+            if hasattr(self, "__name__"):
+                self._raise_attribute_error(name)
+            super().__setattr__(name, value)
+
+        def __delattr__(self, name: str, /) -> Never:
+            self._raise_attribute_error(name)
+
+        def _raise_attribute_error(self, name: str) -> Never:
+            # Match the Python 3.12 error messages exactly
+            if name == "__name__":
+                raise AttributeError("readonly attribute")
+            elif name in {"__value__", "__type_params__", "__parameters__", "__module__"}:
+                raise AttributeError(
+                    f"attribute '{name}' of 'typing.TypeAliasType' objects "
+                    "is not writable"
+                )
+            else:
+                raise AttributeError(
+                    f"'typing.TypeAliasType' object has no attribute '{name}'"
+                )
+
+        def __repr__(self) -> str:
+            return self.__name__
+
+        if sys.version_info < (3, 11):
+            def _check_single_param(self, param, recursion=0):
+                # Allow [], [int], [int, str], [int, ...], [int, T]
+                if param is ...:
+                    return ...
+                if param is None:
+                    return None
+                # Note in <= 3.9 _ConcatenateGenericAlias inherits from list
+                if isinstance(param, list) and recursion == 0:
+                    return [self._check_single_param(arg, recursion+1)
+                            for arg in param]
+                return typing._type_check(
+                        param, f'Subscripting {self.__name__} requires a type.'
+                    )
+
+        def _check_parameters(self, parameters):
+            if sys.version_info < (3, 11):
+                return tuple(
+                    self._check_single_param(item)
+                    for item in parameters
+                )
+            return tuple(typing._type_check(
+                        item, f'Subscripting {self.__name__} requires a type.'
+                    )
+                    for item in parameters
+            )
+
+        def __getitem__(self, parameters):
+            if not self.__type_params__:
+                raise TypeError("Only generic type aliases are subscriptable")
+            if not isinstance(parameters, tuple):
+                parameters = (parameters,)
+            # Using 3.9 here will create problems with Concatenate
+            if sys.version_info >= (3, 10):
+                return _types.GenericAlias(self, parameters)
+            type_vars = _collect_type_vars(parameters)
+            parameters = self._check_parameters(parameters)
+            alias = _TypeAliasGenericAlias(self, parameters)
+            # alias.__parameters__ is not complete if Concatenate is present
+            # as it is converted to a list from which no parameters are extracted.
+            if alias.__parameters__ != type_vars:
+                alias.__parameters__ = type_vars
+            return alias
+
+        def __reduce__(self):
+            return self.__name__
+
+        def __init_subclass__(cls, *args, **kwargs):
+            raise TypeError(
+                "type 'typing_extensions.TypeAliasType' is not an acceptable base type"
+            )
+
+        # The presence of this method convinces typing._type_check
+        # that TypeAliasTypes are types.
+        def __call__(self):
+            raise TypeError("Type alias is not callable")
+
+        if sys.version_info >= (3, 10):
+            def __or__(self, right):
+                # For forward compatibility with 3.12, reject Unions
+                # that are not accepted by the built-in Union.
+                if not _is_unionable(right):
+                    return NotImplemented
+                return typing.Union[self, right]
+
+            def __ror__(self, left):
+                if not _is_unionable(left):
+                    return NotImplemented
+                return typing.Union[left, self]
+
+
+if hasattr(typing, "is_protocol"):
+    is_protocol = typing.is_protocol
+    get_protocol_members = typing.get_protocol_members
+else:
+    def is_protocol(tp: type, /) -> bool:
+        """Return True if the given type is a Protocol.
+
+        Example::
+
+            >>> from typing_extensions import Protocol, is_protocol
+            >>> class P(Protocol):
+            ...     def a(self) -> str: ...
+            ...     b: int
+            >>> is_protocol(P)
+            True
+            >>> is_protocol(int)
+            False
+        """
+        return (
+            isinstance(tp, type)
+            and getattr(tp, '_is_protocol', False)
+            and tp is not Protocol
+            and tp is not typing.Protocol
+        )
+
+    def get_protocol_members(tp: type, /) -> typing.FrozenSet[str]:
+        """Return the set of members defined in a Protocol.
+
+        Example::
+
+            >>> from typing_extensions import Protocol, get_protocol_members
+            >>> class P(Protocol):
+            ...     def a(self) -> str: ...
+            ...     b: int
+            >>> get_protocol_members(P)
+            frozenset({'a', 'b'})
+
+        Raise a TypeError for arguments that are not Protocols.
+        """
+        if not is_protocol(tp):
+            raise TypeError(f'{tp!r} is not a Protocol')
+        if hasattr(tp, '__protocol_attrs__'):
+            return frozenset(tp.__protocol_attrs__)
+        return frozenset(_get_protocol_attrs(tp))
+
+
+if hasattr(typing, "Doc"):
+    Doc = typing.Doc
+else:
+    class Doc:
+        """Define the documentation of a type annotation using ``Annotated``, to be
+         used in class attributes, function and method parameters, return values,
+         and variables.
+
+        The value should be a positional-only string literal to allow static tools
+        like editors and documentation generators to use it.
+
+        This complements docstrings.
+
+        The string value passed is available in the attribute ``documentation``.
+
+        Example::
+
+            >>> from typing_extensions import Annotated, Doc
+            >>> def hi(to: Annotated[str, Doc("Who to say hi to")]) -> None: ...
+        """
+        def __init__(self, documentation: str, /) -> None:
+            self.documentation = documentation
+
+        def __repr__(self) -> str:
+            return f"Doc({self.documentation!r})"
+
+        def __hash__(self) -> int:
+            return hash(self.documentation)
+
+        def __eq__(self, other: object) -> bool:
+            if not isinstance(other, Doc):
+                return NotImplemented
+            return self.documentation == other.documentation
+
+
+_CapsuleType = getattr(_types, "CapsuleType", None)
+
+if _CapsuleType is None:
+    try:
+        import _socket
+    except ImportError:
+        pass
+    else:
+        _CAPI = getattr(_socket, "CAPI", None)
+        if _CAPI is not None:
+            _CapsuleType = type(_CAPI)
+
+if _CapsuleType is not None:
+    CapsuleType = _CapsuleType
+    __all__.append("CapsuleType")
+
+
+if sys.version_info >= (3,14):
+    from annotationlib import Format, get_annotations
+else:
+    class Format(enum.IntEnum):
+        VALUE = 1
+        VALUE_WITH_FAKE_GLOBALS = 2
+        FORWARDREF = 3
+        STRING = 4
+
+    def get_annotations(obj, *, globals=None, locals=None, eval_str=False,
+                        format=Format.VALUE):
+        """Compute the annotations dict for an object.
+
+        obj may be a callable, class, or module.
+        Passing in an object of any other type raises TypeError.
+
+        Returns a dict.  get_annotations() returns a new dict every time
+        it's called; calling it twice on the same object will return two
+        different but equivalent dicts.
+
+        This is a backport of `inspect.get_annotations`, which has been
+        in the standard library since Python 3.10. See the standard library
+        documentation for more:
+
+            https://docs.python.org/3/library/inspect.html#inspect.get_annotations
+
+        This backport adds the *format* argument introduced by PEP 649. The
+        three formats supported are:
+        * VALUE: the annotations are returned as-is. This is the default and
+          it is compatible with the behavior on previous Python versions.
+        * FORWARDREF: return annotations as-is if possible, but replace any
+          undefined names with ForwardRef objects. The implementation proposed by
+          PEP 649 relies on language changes that cannot be backported; the
+          typing-extensions implementation simply returns the same result as VALUE.
+        * STRING: return annotations as strings, in a format close to the original
+          source. Again, this behavior cannot be replicated directly in a backport.
+          As an approximation, typing-extensions retrieves the annotations under
+          VALUE semantics and then stringifies them.
+
+        The purpose of this backport is to allow users who would like to use
+        FORWARDREF or STRING semantics once PEP 649 is implemented, but who also
+        want to support earlier Python versions, to simply write:
+
+            typing_extensions.get_annotations(obj, format=Format.FORWARDREF)
+
+        """
+        format = Format(format)
+        if format is Format.VALUE_WITH_FAKE_GLOBALS:
+            raise ValueError(
+                "The VALUE_WITH_FAKE_GLOBALS format is for internal use only"
+            )
+
+        if eval_str and format is not Format.VALUE:
+            raise ValueError("eval_str=True is only supported with format=Format.VALUE")
+
+        if isinstance(obj, type):
+            # class
+            obj_dict = getattr(obj, '__dict__', None)
+            if obj_dict and hasattr(obj_dict, 'get'):
+                ann = obj_dict.get('__annotations__', None)
+                if isinstance(ann, _types.GetSetDescriptorType):
+                    ann = None
+            else:
+                ann = None
+
+            obj_globals = None
+            module_name = getattr(obj, '__module__', None)
+            if module_name:
+                module = sys.modules.get(module_name, None)
+                if module:
+                    obj_globals = getattr(module, '__dict__', None)
+            obj_locals = dict(vars(obj))
+            unwrap = obj
+        elif isinstance(obj, _types.ModuleType):
+            # module
+            ann = getattr(obj, '__annotations__', None)
+            obj_globals = obj.__dict__
+            obj_locals = None
+            unwrap = None
+        elif callable(obj):
+            # this includes types.Function, types.BuiltinFunctionType,
+            # types.BuiltinMethodType, functools.partial, functools.singledispatch,
+            # "class funclike" from Lib/test/test_inspect... on and on it goes.
+            ann = getattr(obj, '__annotations__', None)
+            obj_globals = getattr(obj, '__globals__', None)
+            obj_locals = None
+            unwrap = obj
+        elif hasattr(obj, '__annotations__'):
+            ann = obj.__annotations__
+            obj_globals = obj_locals = unwrap = None
+        else:
+            raise TypeError(f"{obj!r} is not a module, class, or callable.")
+
+        if ann is None:
+            return {}
+
+        if not isinstance(ann, dict):
+            raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None")
+
+        if not ann:
+            return {}
+
+        if not eval_str:
+            if format is Format.STRING:
+                return {
+                    key: value if isinstance(value, str) else typing._type_repr(value)
+                    for key, value in ann.items()
+                }
+            return dict(ann)
+
+        if unwrap is not None:
+            while True:
+                if hasattr(unwrap, '__wrapped__'):
+                    unwrap = unwrap.__wrapped__
+                    continue
+                if isinstance(unwrap, functools.partial):
+                    unwrap = unwrap.func
+                    continue
+                break
+            if hasattr(unwrap, "__globals__"):
+                obj_globals = unwrap.__globals__
+
+        if globals is None:
+            globals = obj_globals
+        if locals is None:
+            locals = obj_locals or {}
+
+        # "Inject" type parameters into the local namespace
+        # (unless they are shadowed by assignments *in* the local namespace),
+        # as a way of emulating annotation scopes when calling `eval()`
+        if type_params := getattr(obj, "__type_params__", ()):
+            locals = {param.__name__: param for param in type_params} | locals
+
+        return_value = {key:
+            value if not isinstance(value, str) else eval(value, globals, locals)
+            for key, value in ann.items() }
+        return return_value
+
+
+if hasattr(typing, "evaluate_forward_ref"):
+    evaluate_forward_ref = typing.evaluate_forward_ref
+else:
+    # Implements annotationlib.ForwardRef.evaluate
+    def _eval_with_owner(
+        forward_ref, *, owner=None, globals=None, locals=None, type_params=None
+    ):
+        if forward_ref.__forward_evaluated__:
+            return forward_ref.__forward_value__
+        if getattr(forward_ref, "__cell__", None) is not None:
+            try:
+                value = forward_ref.__cell__.cell_contents
+            except ValueError:
+                pass
+            else:
+                forward_ref.__forward_evaluated__ = True
+                forward_ref.__forward_value__ = value
+                return value
+        if owner is None:
+            owner = getattr(forward_ref, "__owner__", None)
+
+        if (
+            globals is None
+            and getattr(forward_ref, "__forward_module__", None) is not None
+        ):
+            globals = getattr(
+                sys.modules.get(forward_ref.__forward_module__, None), "__dict__", None
+            )
+        if globals is None:
+            globals = getattr(forward_ref, "__globals__", None)
+        if globals is None:
+            if isinstance(owner, type):
+                module_name = getattr(owner, "__module__", None)
+                if module_name:
+                    module = sys.modules.get(module_name, None)
+                    if module:
+                        globals = getattr(module, "__dict__", None)
+            elif isinstance(owner, _types.ModuleType):
+                globals = getattr(owner, "__dict__", None)
+            elif callable(owner):
+                globals = getattr(owner, "__globals__", None)
+
+        # If we pass None to eval() below, the globals of this module are used.
+        if globals is None:
+            globals = {}
+
+        if locals is None:
+            locals = {}
+            if isinstance(owner, type):
+                locals.update(vars(owner))
+
+        if type_params is None and owner is not None:
+            # "Inject" type parameters into the local namespace
+            # (unless they are shadowed by assignments *in* the local namespace),
+            # as a way of emulating annotation scopes when calling `eval()`
+            type_params = getattr(owner, "__type_params__", None)
+
+        # type parameters require some special handling,
+        # as they exist in their own scope
+        # but `eval()` does not have a dedicated parameter for that scope.
+        # For classes, names in type parameter scopes should override
+        # names in the global scope (which here are called `localns`!),
+        # but should in turn be overridden by names in the class scope
+        # (which here are called `globalns`!)
+        if type_params is not None:
+            globals = dict(globals)
+            locals = dict(locals)
+            for param in type_params:
+                param_name = param.__name__
+                if (
+                    _FORWARD_REF_HAS_CLASS and not forward_ref.__forward_is_class__
+                ) or param_name not in globals:
+                    globals[param_name] = param
+                    locals.pop(param_name, None)
+
+        arg = forward_ref.__forward_arg__
+        if arg.isidentifier() and not keyword.iskeyword(arg):
+            if arg in locals:
+                value = locals[arg]
+            elif arg in globals:
+                value = globals[arg]
+            elif hasattr(builtins, arg):
+                return getattr(builtins, arg)
+            else:
+                raise NameError(arg)
+        else:
+            code = forward_ref.__forward_code__
+            value = eval(code, globals, locals)
+        forward_ref.__forward_evaluated__ = True
+        forward_ref.__forward_value__ = value
+        return value
+
+    def evaluate_forward_ref(
+        forward_ref,
+        *,
+        owner=None,
+        globals=None,
+        locals=None,
+        type_params=None,
+        format=None,
+        _recursive_guard=frozenset(),
+    ):
+        """Evaluate a forward reference as a type hint.
+
+        This is similar to calling the ForwardRef.evaluate() method,
+        but unlike that method, evaluate_forward_ref() also:
+
+        * Recursively evaluates forward references nested within the type hint.
+        * Rejects certain objects that are not valid type hints.
+        * Replaces type hints that evaluate to None with types.NoneType.
+        * Supports the *FORWARDREF* and *STRING* formats.
+
+        *forward_ref* must be an instance of ForwardRef. *owner*, if given,
+        should be the object that holds the annotations that the forward reference
+        derived from, such as a module, class object, or function. It is used to
+        infer the namespaces to use for looking up names. *globals* and *locals*
+        can also be explicitly given to provide the global and local namespaces.
+        *type_params* is a tuple of type parameters that are in scope when
+        evaluating the forward reference. This parameter must be provided (though
+        it may be an empty tuple) if *owner* is not given and the forward reference
+        does not already have an owner set. *format* specifies the format of the
+        annotation and is a member of the annotationlib.Format enum.
+
+        """
+        if format == Format.STRING:
+            return forward_ref.__forward_arg__
+        if forward_ref.__forward_arg__ in _recursive_guard:
+            return forward_ref
+
+        # Evaluate the forward reference
+        try:
+            value = _eval_with_owner(
+                forward_ref,
+                owner=owner,
+                globals=globals,
+                locals=locals,
+                type_params=type_params,
+            )
+        except NameError:
+            if format == Format.FORWARDREF:
+                return forward_ref
+            else:
+                raise
+
+        if isinstance(value, str):
+            value = ForwardRef(value)
+
+        # Recursively evaluate the type
+        if isinstance(value, ForwardRef):
+            if getattr(value, "__forward_module__", True) is not None:
+                globals = None
+            return evaluate_forward_ref(
+                value,
+                globals=globals,
+                locals=locals,
+                 type_params=type_params, owner=owner,
+                _recursive_guard=_recursive_guard, format=format
+            )
+        if sys.version_info < (3, 12, 5) and type_params:
+            # Make use of type_params
+            locals = dict(locals) if locals else {}
+            for tvar in type_params:
+                if tvar.__name__ not in locals:  # lets not overwrite something present
+                    locals[tvar.__name__] = tvar
+        if sys.version_info < (3, 12, 5):
+            return typing._eval_type(
+                value,
+                globals,
+                locals,
+                recursive_guard=_recursive_guard | {forward_ref.__forward_arg__},
+            )
+        else:
+            return typing._eval_type(
+                value,
+                globals,
+                locals,
+                type_params,
+                recursive_guard=_recursive_guard | {forward_ref.__forward_arg__},
+            )
+
+
+class Sentinel:
+    """Create a unique sentinel object.
+
+    *name* should be the name of the variable to which the return value shall be assigned.
+
+    *repr*, if supplied, will be used for the repr of the sentinel object.
+    If not provided, "" will be used.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        repr: typing.Optional[str] = None,
+    ):
+        self._name = name
+        self._repr = repr if repr is not None else f'<{name}>'
+
+    def __repr__(self):
+        return self._repr
+
+    if sys.version_info < (3, 11):
+        # The presence of this method convinces typing._type_check
+        # that Sentinels are types.
+        def __call__(self, *args, **kwargs):
+            raise TypeError(f"{type(self).__name__!r} object is not callable")
+
+    if sys.version_info >= (3, 10):
+        def __or__(self, other):
+            return typing.Union[self, other]
+
+        def __ror__(self, other):
+            return typing.Union[other, self]
+
+    def __getstate__(self):
+        raise TypeError(f"Cannot pickle {type(self).__name__!r} object")
+
+
+# Aliases for items that are in typing in all supported versions.
+# We use hasattr() checks so this library will continue to import on
+# future versions of Python that may remove these names.
+_typing_names = [
+    "AbstractSet",
+    "AnyStr",
+    "BinaryIO",
+    "Callable",
+    "Collection",
+    "Container",
+    "Dict",
+    "FrozenSet",
+    "Hashable",
+    "IO",
+    "ItemsView",
+    "Iterable",
+    "Iterator",
+    "KeysView",
+    "List",
+    "Mapping",
+    "MappingView",
+    "Match",
+    "MutableMapping",
+    "MutableSequence",
+    "MutableSet",
+    "Optional",
+    "Pattern",
+    "Reversible",
+    "Sequence",
+    "Set",
+    "Sized",
+    "TextIO",
+    "Tuple",
+    "Union",
+    "ValuesView",
+    "cast",
+    "no_type_check",
+    "no_type_check_decorator",
+    # This is private, but it was defined by typing_extensions for a long time
+    # and some users rely on it.
+    "_AnnotatedAlias",
+]
+globals().update(
+    {name: getattr(typing, name) for name in _typing_names if hasattr(typing, name)}
+)
+# These are defined unconditionally because they are used in
+# typing-extensions itself.
+Generic = typing.Generic
+ForwardRef = typing.ForwardRef
+Annotated = typing.Annotated
diff --git a/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/INSTALLER b/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/INSTALLER
new file mode 100644
index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68
--- /dev/null
+++ b/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/METADATA b/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/METADATA
new file mode 100644
index 0000000000000000000000000000000000000000..1640d911c8a23ed7dca476ed58a962b1923e4473
--- /dev/null
+++ b/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/METADATA
@@ -0,0 +1,70 @@
+Metadata-Version: 2.1
+Name: xgrammar
+Version: 0.1.18
+Summary: Efficient, Flexible and Portable Structured Generation
+Keywords: machine learning,inference
+Author: MLC Team
+License: Apache 2.0
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Project-URL: Homepage, https://xgrammar.mlc.ai/
+Project-URL: GitHub, https://github.com/mlc-ai/xgrammar
+Requires-Python: <4,>=3.8
+Requires-Dist: pydantic
+Requires-Dist: sentencepiece
+Requires-Dist: tiktoken
+Requires-Dist: torch>=1.10.0
+Requires-Dist: transformers>=4.38.0
+Requires-Dist: triton; platform_system == "Linux" and platform_machine == "x86_64"
+Requires-Dist: mlx-lm; platform_system == "Darwin" and platform_machine == "arm64"
+Requires-Dist: ninja
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: protobuf; extra == "test"
+Requires-Dist: huggingface-hub[cli]; extra == "test"
+Requires-Dist: transformers<4.50.0; platform_system == "Darwin" and extra == "test"
+Description-Content-Type: text/markdown
+
+
    
+
+# XGrammar
+
+[![Documentation](https://img.shields.io/badge/docs-latest-green)](https://xgrammar.mlc.ai/docs/)
+[![License](https://img.shields.io/badge/license-apache_2-blue)](https://github.com/mlc-ai/xgrammar/blob/main/LICENSE)
+[![PyPI](https://img.shields.io/pypi/v/xgrammar)](https://pypi.org/project/xgrammar)
+[![PyPI Downloads](https://static.pepy.tech/badge/xgrammar)](https://pepy.tech/projects/xgrammar)
+
+**Efficient, Flexible and Portable Structured Generation**
+
+
+[Get Started](#get-started) | [Documentation](https://xgrammar.mlc.ai/docs/) | [Blogpost](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar) | [Technical Report](https://arxiv.org/abs/2411.15100)
+
+
    
+
+## News
+- [2025/02] XGrammar has been officially integrated into [Modular's MAX](https://docs.modular.com/max/serve/structured-output)
+- [2025/01] XGrammar has been officially integrated into [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM).
+- [2024/12] XGrammar has been officially integrated into [vLLM](https://github.com/vllm-project/vllm).
+- [2024/12] We presented research talks on XGrammar at CMU Catalyst, Berkeley SkyLab, MIT HANLAB, THU IIIS, SJTU, Ant Group, SGLang Meetup, Qingke AI, Camel AI. The slides can be found [here](https://docs.google.com/presentation/d/1iS7tu2EV4IKRWDaR0F3YD7ubrNqtGYUStSskceneelc/edit?usp=sharing).
+- [2024/11] XGrammar has been officially integrated into [SGLang](https://github.com/sgl-project/sglang).
+- [2024/11] XGrammar has been officially integrated into [MLC-LLM](https://github.com/mlc-ai/mlc-llm).
+- [2024/11] We officially released XGrammar v0.1.0!
+
+## Overview
+
+
+XGrammar is an open-source library for efficient, flexible, and portable structured generation.
+It supports general context-free grammar to enable a broad range of structures while bringing careful system optimizations to enable fast executions.
+XGrammar features a minimal and portable C++ backend that can be easily integrated into multiple environments and frameworks,
+and is co-designed with the LLM inference engine and enables zero-overhead structured generation in LLM inference.
+
+
+
+## Get Started
+
+Please visit our [documentation](https://xgrammar.mlc.ai/docs/) to get started with XGrammar.
+- [Installation](https://xgrammar.mlc.ai/docs/start/install)
+- [Quick start](https://xgrammar.mlc.ai/docs/start/quick_start)
diff --git a/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/RECORD b/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/RECORD
new file mode 100644
index 0000000000000000000000000000000000000000..c26be7a7809eb11b21343594ed3fc53fa6a6e43f
--- /dev/null
+++ b/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/RECORD
@@ -0,0 +1,46 @@
+xgrammar-0.1.18.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+xgrammar-0.1.18.dist-info/METADATA,sha256=gknu2i0vD8g7MaC-gSTwOdDxJcyjPDRFVKjel1izUq0,3584
+xgrammar-0.1.18.dist-info/RECORD,,
+xgrammar-0.1.18.dist-info/WHEEL,sha256=eyOW1G9YDUewwwRJOGy55SqofaokBV2FtqXqzhzU_cA,156
+xgrammar-0.1.18.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+xgrammar-0.1.18.dist-info/licenses/NOTICE,sha256=dpxpBwc8ZYAb_P484cDoC_CuThJOJ2nU8CbAOK1YPA8,54
+xgrammar/__init__.py,sha256=vr-8u4CcpaFRx89GPv1Y2Pyjzudo6zDuwYs9Dz0ZVP8,375
+xgrammar/__pycache__/__init__.cpython-310.pyc,,
+xgrammar/__pycache__/base.cpython-310.pyc,,
+xgrammar/__pycache__/compiler.cpython-310.pyc,,
+xgrammar/__pycache__/grammar.cpython-310.pyc,,
+xgrammar/__pycache__/matcher.cpython-310.pyc,,
+xgrammar/__pycache__/testing.cpython-310.pyc,,
+xgrammar/__pycache__/tokenizer_info.cpython-310.pyc,,
+xgrammar/__pycache__/version.cpython-310.pyc,,
+xgrammar/base.py,sha256=P1vjpdTGijQGy25bKiawFCqMcxYm_HqAO7WFUjiiuJ8,2360
+xgrammar/compiler.py,sha256=UCb9Fol_NBCFTDXhimc-J3P2X8kgkNrNWVflBoL1TqA,7728
+xgrammar/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xgrammar/contrib/__pycache__/__init__.cpython-310.pyc,,
+xgrammar/contrib/__pycache__/hf.cpython-310.pyc,,
+xgrammar/contrib/__pycache__/mlxlm.cpython-310.pyc,,
+xgrammar/contrib/hf.py,sha256=nAoqTdxWhP7uWEba1HAsSyrn1MYZYvKRudDkrerIH-k,4614
+xgrammar/contrib/mlxlm.py,sha256=eoZ5baxaIJplP0ohiBY6AjofHbjfn6J5v9sFWi-lwpw,2812
+xgrammar/grammar.py,sha256=VgyNnAZFhLHHejO2W9OgWtn7ON5l7iitpBVxyestL3Y,13072
+xgrammar/kernels/__init__.py,sha256=Rz8sBcZMbQeYZiKoxvG1u7jteQj0S7gOJ3MCOfJWRX0,259
+xgrammar/kernels/__pycache__/__init__.cpython-310.pyc,,
+xgrammar/kernels/__pycache__/apply_token_bitmask_inplace_cpu.cpython-310.pyc,,
+xgrammar/kernels/__pycache__/apply_token_bitmask_inplace_cuda.cpython-310.pyc,,
+xgrammar/kernels/__pycache__/apply_token_bitmask_inplace_torch_compile.cpython-310.pyc,,
+xgrammar/kernels/__pycache__/apply_token_bitmask_inplace_triton.cpython-310.pyc,,
+xgrammar/kernels/__pycache__/apply_token_bitmask_mlx.cpython-310.pyc,,
+xgrammar/kernels/apply_token_bitmask_inplace_cpu.py,sha256=HOxSWkD4IZ3ikKDYKUPsaz-DL2LU8-69y_msX5tpf9k,1512
+xgrammar/kernels/apply_token_bitmask_inplace_cuda.cu,sha256=jEEl3KqavKcPDsRkf1AcPVwlaTUeVO8oOT7EX7rdQbU,10015
+xgrammar/kernels/apply_token_bitmask_inplace_cuda.py,sha256=m5bLvWDtXQG0t4FTY8akh_6NOD0hDTyp7ENN1Smrvk8,4049
+xgrammar/kernels/apply_token_bitmask_inplace_torch_compile.py,sha256=m2-wTBVvSyi-sebEgSKRfQIhg8_0jt5ZhgW-dPKWjs4,2138
+xgrammar/kernels/apply_token_bitmask_inplace_triton.py,sha256=a4STzJf7CKLZayqsz_Hqyo-zlqXkdk2Fmuqc6BZi73A,3810
+xgrammar/kernels/apply_token_bitmask_mlx.py,sha256=N1hbWiCyc0eOuK0R2Cj9LHWVi1DnOqZjWVP5TvQlgsU,883
+xgrammar/matcher.py,sha256=HrVDW_IDIl2OTV8N7TKED0Ez9uc4lTIEfs7jfBxP4Rg,13017
+xgrammar/support/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xgrammar/support/__pycache__/__init__.cpython-310.pyc,,
+xgrammar/support/__pycache__/logging.cpython-310.pyc,,
+xgrammar/support/logging.py,sha256=8jmO9FAVVJpgbLOwclXpAowOe2vjdbtjk3Ryitphl9U,594
+xgrammar/testing.py,sha256=bOYvVYCfVBbAYWWmcc9SJF8zgy7UiEL72B7juaDxhDI,10867
+xgrammar/tokenizer_info.py,sha256=348J7MkyEBtvTUcyE4diby0xPM0U-kGeN9glu7M787Q,15734
+xgrammar/version.py,sha256=7VtgdhF7CEhyWBFHSaTl_LXmNFhWTvlSu3U4f222M_0,4296
+xgrammar/xgrammar_bindings.cpython-310-x86_64-linux-gnu.so,sha256=6YOnnwN5gmowvKM0dvD7-6HEr5su6wfcwD4r2UBL8CI,17243960
diff --git a/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/WHEEL b/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/WHEEL
new file mode 100644
index 0000000000000000000000000000000000000000..f4b427693f086609b0ceab2de98bd44912d9bf21
--- /dev/null
+++ b/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/WHEEL
@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: scikit-build-core 0.11.1
+Root-Is-Purelib: false
+Tag: cp310-cp310-manylinux_2_17_x86_64
+Tag: cp310-cp310-manylinux2014_x86_64
+
diff --git a/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/licenses/LICENSE b/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/licenses/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64
--- /dev/null
+++ b/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/licenses/LICENSE
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/licenses/NOTICE b/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/licenses/NOTICE
new file mode 100644
index 0000000000000000000000000000000000000000..2d56df4754f7ebeba356393e69df5e2eef265232
--- /dev/null
+++ b/lib/python3.1/site-packages/xgrammar-0.1.18.dist-info/licenses/NOTICE
@@ -0,0 +1,3 @@
+XGrammar
+
+Copyright (c) 2024 by XGrammar Contributors
diff --git a/lib/python3.1/turtledemo/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/turtledemo/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6eb056eae4544af7d28d6ccca1eff699f2f8ffaa
Binary files /dev/null and b/lib/python3.1/turtledemo/__pycache__/__init__.cpython-310.pyc differ
diff --git a/lib/python3.1/turtledemo/__pycache__/__main__.cpython-310.pyc b/lib/python3.1/turtledemo/__pycache__/__main__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4a3d3a6a81a588c418024f451a13fb813c4ec96d
Binary files /dev/null and b/lib/python3.1/turtledemo/__pycache__/__main__.cpython-310.pyc differ
diff --git a/lib/python3.1/turtledemo/__pycache__/bytedesign.cpython-310.pyc b/lib/python3.1/turtledemo/__pycache__/bytedesign.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0a97a43bb4be6062d5cbb0348dffeb5e1824fad7
Binary files /dev/null and b/lib/python3.1/turtledemo/__pycache__/bytedesign.cpython-310.pyc differ
diff --git a/lib/python3.1/turtledemo/__pycache__/colormixer.cpython-310.pyc b/lib/python3.1/turtledemo/__pycache__/colormixer.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..dfcb80f0ab2686d6617545d3375502e9eed14861
Binary files /dev/null and b/lib/python3.1/turtledemo/__pycache__/colormixer.cpython-310.pyc differ
diff --git a/lib/python3.1/turtledemo/__pycache__/fractalcurves.cpython-310.pyc b/lib/python3.1/turtledemo/__pycache__/fractalcurves.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1a2d4c267f419f32bd9ef98b68cc83aef9b226a4
Binary files /dev/null and b/lib/python3.1/turtledemo/__pycache__/fractalcurves.cpython-310.pyc differ
diff --git a/lib/python3.1/turtledemo/__pycache__/minimal_hanoi.cpython-310.pyc b/lib/python3.1/turtledemo/__pycache__/minimal_hanoi.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e478db30203c511788eaa0f57ced40eab4a12ab6
Binary files /dev/null and b/lib/python3.1/turtledemo/__pycache__/minimal_hanoi.cpython-310.pyc differ
diff --git a/lib/python3.1/turtledemo/__pycache__/nim.cpython-310.pyc b/lib/python3.1/turtledemo/__pycache__/nim.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0b32f7da0576f9e0b941b1ad0ca81e4aaa3807b5
Binary files /dev/null and b/lib/python3.1/turtledemo/__pycache__/nim.cpython-310.pyc differ
diff --git a/lib/python3.1/turtledemo/__pycache__/peace.cpython-310.pyc b/lib/python3.1/turtledemo/__pycache__/peace.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cb216a630fc6916cd0f156e4337763a64643ff66
Binary files /dev/null and b/lib/python3.1/turtledemo/__pycache__/peace.cpython-310.pyc differ
diff --git a/lib/python3.1/turtledemo/__pycache__/rosette.cpython-310.pyc b/lib/python3.1/turtledemo/__pycache__/rosette.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b2e10c6fa0c445d25286adc66f42c01ded760ccd
Binary files /dev/null and b/lib/python3.1/turtledemo/__pycache__/rosette.cpython-310.pyc differ
diff --git a/lib/python3.1/turtledemo/__pycache__/round_dance.cpython-310.pyc b/lib/python3.1/turtledemo/__pycache__/round_dance.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..dbd432e1192adfc64d87c49d3de970056ec23dc6
Binary files /dev/null and b/lib/python3.1/turtledemo/__pycache__/round_dance.cpython-310.pyc differ
diff --git a/lib/python3.1/turtledemo/__pycache__/tree.cpython-310.pyc b/lib/python3.1/turtledemo/__pycache__/tree.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..064b49f9eb034d6ec2644ea6e3b6708a4a50d3f7
Binary files /dev/null and b/lib/python3.1/turtledemo/__pycache__/tree.cpython-310.pyc differ
diff --git a/lib/python3.1/turtledemo/__pycache__/two_canvases.cpython-310.pyc b/lib/python3.1/turtledemo/__pycache__/two_canvases.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..763ae0699e86fa9a2d1175a6698516761b2f65c3
Binary files /dev/null and b/lib/python3.1/turtledemo/__pycache__/two_canvases.cpython-310.pyc differ
diff --git a/lib/python3.1/turtledemo/__pycache__/yinyang.cpython-310.pyc b/lib/python3.1/turtledemo/__pycache__/yinyang.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..27110efc537e34b70638d5a0a1f37be5cd007067
Binary files /dev/null and b/lib/python3.1/turtledemo/__pycache__/yinyang.cpython-310.pyc differ
diff --git a/lib/python3.1/urllib/__init__.py b/lib/python3.1/urllib/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lib/python3.1/urllib/__pycache__/__init__.cpython-310.pyc b/lib/python3.1/urllib/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b9ce19f3d4a090be62a804c84c2e6a49f5bce998
Binary files /dev/null and b/lib/python3.1/urllib/__pycache__/__init__.cpython-310.pyc differ
diff --git a/lib/python3.1/urllib/__pycache__/error.cpython-310.pyc b/lib/python3.1/urllib/__pycache__/error.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ae65e7786549e3d00f3528237e1bf1157e8aa2cb
Binary files /dev/null and b/lib/python3.1/urllib/__pycache__/error.cpython-310.pyc differ
diff --git a/lib/python3.1/urllib/__pycache__/parse.cpython-310.pyc b/lib/python3.1/urllib/__pycache__/parse.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7bda333fccf8efb294d2b9c5cff9408a762bf954
Binary files /dev/null and b/lib/python3.1/urllib/__pycache__/parse.cpython-310.pyc differ
diff --git a/lib/python3.1/urllib/__pycache__/request.cpython-310.pyc b/lib/python3.1/urllib/__pycache__/request.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..52c80aa641bcfa2325c38ffc17886ea1ab0c9d55
Binary files /dev/null and b/lib/python3.1/urllib/__pycache__/request.cpython-310.pyc differ
diff --git a/lib/python3.1/urllib/__pycache__/response.cpython-310.pyc b/lib/python3.1/urllib/__pycache__/response.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a5ca66ab119958504091983598df6f6a2eaf0837
Binary files /dev/null and b/lib/python3.1/urllib/__pycache__/response.cpython-310.pyc differ
diff --git a/lib/python3.1/urllib/__pycache__/robotparser.cpython-310.pyc b/lib/python3.1/urllib/__pycache__/robotparser.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..46e43b0044745fdfc47ede4b7170173327e699ae
Binary files /dev/null and b/lib/python3.1/urllib/__pycache__/robotparser.cpython-310.pyc differ
diff --git a/lib/python3.1/urllib/error.py b/lib/python3.1/urllib/error.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9cd1ecadd63f2b09ee4363fc283e8f1f5cbbb3b
--- /dev/null
+++ b/lib/python3.1/urllib/error.py
@@ -0,0 +1,74 @@
+"""Exception classes raised by urllib.
+
+The base exception class is URLError, which inherits from OSError.  It
+doesn't define any behavior of its own, but is the base class for all
+exceptions defined in this package.
+
+HTTPError is an exception class that is also a valid HTTP response
+instance.  It behaves this way because HTTP protocol errors are valid
+responses, with a status code, headers, and a body.  In some contexts,
+an application may want to handle an exception like a regular
+response.
+"""
+import io
+import urllib.response
+
+__all__ = ['URLError', 'HTTPError', 'ContentTooShortError']
+
+
+class URLError(OSError):
+    # URLError is a sub-type of OSError, but it doesn't share any of
+    # the implementation.  need to override __init__ and __str__.
+    # It sets self.args for compatibility with other OSError
+    # subclasses, but args doesn't have the typical format with errno in
+    # slot 0 and strerror in slot 1.  This may be better than nothing.
+    def __init__(self, reason, filename=None):
+        self.args = reason,
+        self.reason = reason
+        if filename is not None:
+            self.filename = filename
+
+    def __str__(self):
+        return '' % self.reason
+
+
+class HTTPError(URLError, urllib.response.addinfourl):
+    """Raised when HTTP error occurs, but also acts like non-error return"""
+    __super_init = urllib.response.addinfourl.__init__
+
+    def __init__(self, url, code, msg, hdrs, fp):
+        self.code = code
+        self.msg = msg
+        self.hdrs = hdrs
+        self.fp = fp
+        self.filename = url
+        if fp is None:
+            fp = io.BytesIO()
+        self.__super_init(fp, hdrs, url, code)
+
+    def __str__(self):
+        return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+    def __repr__(self):
+        return '' % (self.code, self.msg)
+
+    # since URLError specifies a .reason attribute, HTTPError should also
+    #  provide this attribute. See issue13211 for discussion.
+    @property
+    def reason(self):
+        return self.msg
+
+    @property
+    def headers(self):
+        return self.hdrs
+
+    @headers.setter
+    def headers(self, headers):
+        self.hdrs = headers
+
+
+class ContentTooShortError(URLError):
+    """Exception raised when downloaded size does not match content-length."""
+    def __init__(self, message, content):
+        URLError.__init__(self, message)
+        self.content = content
diff --git a/lib/python3.1/urllib/parse.py b/lib/python3.1/urllib/parse.py
new file mode 100644
index 0000000000000000000000000000000000000000..e1ee36d98e71edbe394529aed9bb8e9f14dc2775
--- /dev/null
+++ b/lib/python3.1/urllib/parse.py
@@ -0,0 +1,1237 @@
+"""Parse (absolute and relative) URLs.
+
+urlparse module is based upon the following RFC specifications.
+
+RFC 3986 (STD66): "Uniform Resource Identifiers" by T. Berners-Lee, R. Fielding
+and L.  Masinter, January 2005.
+
+RFC 2732 : "Format for Literal IPv6 Addresses in URL's by R.Hinden, B.Carpenter
+and L.Masinter, December 1999.
+
+RFC 2396:  "Uniform Resource Identifiers (URI)": Generic Syntax by T.
+Berners-Lee, R. Fielding, and L. Masinter, August 1998.
+
+RFC 2368: "The mailto URL scheme", by P.Hoffman , L Masinter, J. Zawinski, July 1998.
+
+RFC 1808: "Relative Uniform Resource Locators", by R. Fielding, UC Irvine, June
+1995.
+
+RFC 1738: "Uniform Resource Locators (URL)" by T. Berners-Lee, L. Masinter, M.
+McCahill, December 1994
+
+RFC 3986 is considered the current standard and any future changes to
+urlparse module should conform with it.  The urlparse module is
+currently not entirely compliant with this RFC due to defacto
+scenarios for parsing, and for backward compatibility purposes, some
+parsing quirks from older RFCs are retained. The testcases in
+test_urlparse.py provides a good indicator of parsing behavior.
+
+The WHATWG URL Parser spec should also be considered.  We are not compliant with
+it either due to existing user code API behavior expectations (Hyrum's Law).
+It serves as a useful guide when making changes.
+"""
+
+import re
+import sys
+import types
+import collections
+import warnings
+import ipaddress
+
+__all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag",
+           "urlsplit", "urlunsplit", "urlencode", "parse_qs",
+           "parse_qsl", "quote", "quote_plus", "quote_from_bytes",
+           "unquote", "unquote_plus", "unquote_to_bytes",
+           "DefragResult", "ParseResult", "SplitResult",
+           "DefragResultBytes", "ParseResultBytes", "SplitResultBytes"]
+
+# A classification of schemes.
+# The empty string classifies URLs with no scheme specified,
+# being the default value returned by “urlsplit” and “urlparse”.
+
+uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap',
+                 'wais', 'file', 'https', 'shttp', 'mms',
+                 'prospero', 'rtsp', 'rtspu', 'sftp',
+                 'svn', 'svn+ssh', 'ws', 'wss']
+
+uses_netloc = ['', 'ftp', 'http', 'gopher', 'nntp', 'telnet',
+               'imap', 'wais', 'file', 'mms', 'https', 'shttp',
+               'snews', 'prospero', 'rtsp', 'rtspu', 'rsync',
+               'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh',
+               'ws', 'wss']
+
+uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap',
+               'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips',
+               'mms', 'sftp', 'tel']
+
+# These are not actually used anymore, but should stay for backwards
+# compatibility.  (They are undocumented, but have a public-looking name.)
+
+non_hierarchical = ['gopher', 'hdl', 'mailto', 'news',
+                    'telnet', 'wais', 'imap', 'snews', 'sip', 'sips']
+
+uses_query = ['', 'http', 'wais', 'imap', 'https', 'shttp', 'mms',
+              'gopher', 'rtsp', 'rtspu', 'sip', 'sips']
+
+uses_fragment = ['', 'ftp', 'hdl', 'http', 'gopher', 'news',
+                 'nntp', 'wais', 'https', 'shttp', 'snews',
+                 'file', 'prospero']
+
+# Characters valid in scheme names
+scheme_chars = ('abcdefghijklmnopqrstuvwxyz'
+                'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+                '0123456789'
+                '+-.')
+
+# Leading and trailing C0 control and space to be stripped per WHATWG spec.
+# == "".join([chr(i) for i in range(0, 0x20 + 1)])
+_WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f '
+
+# Unsafe bytes to be removed per WHATWG spec
+_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n']
+
+# XXX: Consider replacing with functools.lru_cache
+MAX_CACHE_SIZE = 20
+_parse_cache = {}
+
+def clear_cache():
+    """Clear the parse cache and the quoters cache."""
+    _parse_cache.clear()
+    _safe_quoters.clear()
+
+
+# Helpers for bytes handling
+# For 3.2, we deliberately require applications that
+# handle improperly quoted URLs to do their own
+# decoding and encoding. If valid use cases are
+# presented, we may relax this by using latin-1
+# decoding internally for 3.3
+_implicit_encoding = 'ascii'
+_implicit_errors = 'strict'
+
+def _noop(obj):
+    return obj
+
+def _encode_result(obj, encoding=_implicit_encoding,
+                        errors=_implicit_errors):
+    return obj.encode(encoding, errors)
+
+def _decode_args(args, encoding=_implicit_encoding,
+                       errors=_implicit_errors):
+    return tuple(x.decode(encoding, errors) if x else '' for x in args)
+
+def _coerce_args(*args):
+    # Invokes decode if necessary to create str args
+    # and returns the coerced inputs along with
+    # an appropriate result coercion function
+    #   - noop for str inputs
+    #   - encoding function otherwise
+    str_input = isinstance(args[0], str)
+    for arg in args[1:]:
+        # We special-case the empty string to support the
+        # "scheme=''" default argument to some functions
+        if arg and isinstance(arg, str) != str_input:
+            raise TypeError("Cannot mix str and non-str arguments")
+    if str_input:
+        return args + (_noop,)
+    return _decode_args(args) + (_encode_result,)
+
+# Result objects are more helpful than simple tuples
+class _ResultMixinStr(object):
+    """Standard approach to encoding parsed results from str to bytes"""
+    __slots__ = ()
+
+    def encode(self, encoding='ascii', errors='strict'):
+        return self._encoded_counterpart(*(x.encode(encoding, errors) for x in self))
+
+
+class _ResultMixinBytes(object):
+    """Standard approach to decoding parsed results from bytes to str"""
+    __slots__ = ()
+
+    def decode(self, encoding='ascii', errors='strict'):
+        return self._decoded_counterpart(*(x.decode(encoding, errors) for x in self))
+
+
+class _NetlocResultMixinBase(object):
+    """Shared methods for the parsed result objects containing a netloc element"""
+    __slots__ = ()
+
+    @property
+    def username(self):
+        return self._userinfo[0]
+
+    @property
+    def password(self):
+        return self._userinfo[1]
+
+    @property
+    def hostname(self):
+        hostname = self._hostinfo[0]
+        if not hostname:
+            return None
+        # Scoped IPv6 address may have zone info, which must not be lowercased
+        # like http://[fe80::822a:a8ff:fe49:470c%tESt]:1234/keys
+        separator = '%' if isinstance(hostname, str) else b'%'
+        hostname, percent, zone = hostname.partition(separator)
+        return hostname.lower() + percent + zone
+
+    @property
+    def port(self):
+        port = self._hostinfo[1]
+        if port is not None:
+            if port.isdigit() and port.isascii():
+                port = int(port)
+            else:
+                raise ValueError(f"Port could not be cast to integer value as {port!r}")
+            if not (0 <= port <= 65535):
+                raise ValueError("Port out of range 0-65535")
+        return port
+
+    __class_getitem__ = classmethod(types.GenericAlias)
+
+
+class _NetlocResultMixinStr(_NetlocResultMixinBase, _ResultMixinStr):
+    __slots__ = ()
+
+    @property
+    def _userinfo(self):
+        netloc = self.netloc
+        userinfo, have_info, hostinfo = netloc.rpartition('@')
+        if have_info:
+            username, have_password, password = userinfo.partition(':')
+            if not have_password:
+                password = None
+        else:
+            username = password = None
+        return username, password
+
+    @property
+    def _hostinfo(self):
+        netloc = self.netloc
+        _, _, hostinfo = netloc.rpartition('@')
+        _, have_open_br, bracketed = hostinfo.partition('[')
+        if have_open_br:
+            hostname, _, port = bracketed.partition(']')
+            _, _, port = port.partition(':')
+        else:
+            hostname, _, port = hostinfo.partition(':')
+        if not port:
+            port = None
+        return hostname, port
+
+
+class _NetlocResultMixinBytes(_NetlocResultMixinBase, _ResultMixinBytes):
+    __slots__ = ()
+
+    @property
+    def _userinfo(self):
+        netloc = self.netloc
+        userinfo, have_info, hostinfo = netloc.rpartition(b'@')
+        if have_info:
+            username, have_password, password = userinfo.partition(b':')
+            if not have_password:
+                password = None
+        else:
+            username = password = None
+        return username, password
+
+    @property
+    def _hostinfo(self):
+        netloc = self.netloc
+        _, _, hostinfo = netloc.rpartition(b'@')
+        _, have_open_br, bracketed = hostinfo.partition(b'[')
+        if have_open_br:
+            hostname, _, port = bracketed.partition(b']')
+            _, _, port = port.partition(b':')
+        else:
+            hostname, _, port = hostinfo.partition(b':')
+        if not port:
+            port = None
+        return hostname, port
+
+
+from collections import namedtuple
+
+_DefragResultBase = namedtuple('DefragResult', 'url fragment')
+_SplitResultBase = namedtuple(
+    'SplitResult', 'scheme netloc path query fragment')
+_ParseResultBase = namedtuple(
+    'ParseResult', 'scheme netloc path params query fragment')
+
+_DefragResultBase.__doc__ = """
+DefragResult(url, fragment)
+
+A 2-tuple that contains the url without fragment identifier and the fragment
+identifier as a separate argument.
+"""
+
+_DefragResultBase.url.__doc__ = """The URL with no fragment identifier."""
+
+_DefragResultBase.fragment.__doc__ = """
+Fragment identifier separated from URL, that allows indirect identification of a
+secondary resource by reference to a primary resource and additional identifying
+information.
+"""
+
+_SplitResultBase.__doc__ = """
+SplitResult(scheme, netloc, path, query, fragment)
+
+A 5-tuple that contains the different components of a URL. Similar to
+ParseResult, but does not split params.
+"""
+
+_SplitResultBase.scheme.__doc__ = """Specifies URL scheme for the request."""
+
+_SplitResultBase.netloc.__doc__ = """
+Network location where the request is made to.
+"""
+
+_SplitResultBase.path.__doc__ = """
+The hierarchical path, such as the path to a file to download.
+"""
+
+_SplitResultBase.query.__doc__ = """
+The query component, that contains non-hierarchical data, that along with data
+in path component, identifies a resource in the scope of URI's scheme and
+network location.
+"""
+
+_SplitResultBase.fragment.__doc__ = """
+Fragment identifier, that allows indirect identification of a secondary resource
+by reference to a primary resource and additional identifying information.
+"""
+
+_ParseResultBase.__doc__ = """
+ParseResult(scheme, netloc, path, params, query, fragment)
+
+A 6-tuple that contains components of a parsed URL.
+"""
+
+_ParseResultBase.scheme.__doc__ = _SplitResultBase.scheme.__doc__
+_ParseResultBase.netloc.__doc__ = _SplitResultBase.netloc.__doc__
+_ParseResultBase.path.__doc__ = _SplitResultBase.path.__doc__
+_ParseResultBase.params.__doc__ = """
+Parameters for last path element used to dereference the URI in order to provide
+access to perform some operation on the resource.
+"""
+
+_ParseResultBase.query.__doc__ = _SplitResultBase.query.__doc__
+_ParseResultBase.fragment.__doc__ = _SplitResultBase.fragment.__doc__
+
+
+# For backwards compatibility, alias _NetlocResultMixinStr
+# ResultBase is no longer part of the documented API, but it is
+# retained since deprecating it isn't worth the hassle
+ResultBase = _NetlocResultMixinStr
+
+# Structured result objects for string data
+class DefragResult(_DefragResultBase, _ResultMixinStr):
+    __slots__ = ()
+    def geturl(self):
+        if self.fragment:
+            return self.url + '#' + self.fragment
+        else:
+            return self.url
+
+class SplitResult(_SplitResultBase, _NetlocResultMixinStr):
+    __slots__ = ()
+    def geturl(self):
+        return urlunsplit(self)
+
+class ParseResult(_ParseResultBase, _NetlocResultMixinStr):
+    __slots__ = ()
+    def geturl(self):
+        return urlunparse(self)
+
+# Structured result objects for bytes data
+class DefragResultBytes(_DefragResultBase, _ResultMixinBytes):
+    __slots__ = ()
+    def geturl(self):
+        if self.fragment:
+            return self.url + b'#' + self.fragment
+        else:
+            return self.url
+
+class SplitResultBytes(_SplitResultBase, _NetlocResultMixinBytes):
+    __slots__ = ()
+    def geturl(self):
+        return urlunsplit(self)
+
+class ParseResultBytes(_ParseResultBase, _NetlocResultMixinBytes):
+    __slots__ = ()
+    def geturl(self):
+        return urlunparse(self)
+
+# Set up the encode/decode result pairs
+def _fix_result_transcoding():
+    _result_pairs = (
+        (DefragResult, DefragResultBytes),
+        (SplitResult, SplitResultBytes),
+        (ParseResult, ParseResultBytes),
+    )
+    for _decoded, _encoded in _result_pairs:
+        _decoded._encoded_counterpart = _encoded
+        _encoded._decoded_counterpart = _decoded
+
+_fix_result_transcoding()
+del _fix_result_transcoding
+
+def urlparse(url, scheme='', allow_fragments=True):
+    """Parse a URL into 6 components:
+    :///;?#
+
+    The result is a named 6-tuple with fields corresponding to the
+    above. It is either a ParseResult or ParseResultBytes object,
+    depending on the type of the url parameter.
+
+    The username, password, hostname, and port sub-components of netloc
+    can also be accessed as attributes of the returned object.
+
+    The scheme argument provides the default value of the scheme
+    component when no scheme is found in url.
+
+    If allow_fragments is False, no attempt is made to separate the
+    fragment component from the previous component, which can be either
+    path or query.
+
+    Note that % escapes are not expanded.
+    """
+    url, scheme, _coerce_result = _coerce_args(url, scheme)
+    splitresult = urlsplit(url, scheme, allow_fragments)
+    scheme, netloc, url, query, fragment = splitresult
+    if scheme in uses_params and ';' in url:
+        url, params = _splitparams(url)
+    else:
+        params = ''
+    result = ParseResult(scheme, netloc, url, params, query, fragment)
+    return _coerce_result(result)
+
+def _splitparams(url):
+    if '/'  in url:
+        i = url.find(';', url.rfind('/'))
+        if i < 0:
+            return url, ''
+    else:
+        i = url.find(';')
+    return url[:i], url[i+1:]
+
+def _splitnetloc(url, start=0):
+    delim = len(url)   # position of end of domain part of url, default is end
+    for c in '/?#':    # look for delimiters; the order is NOT important
+        wdelim = url.find(c, start)        # find first of this delim
+        if wdelim >= 0:                    # if found
+            delim = min(delim, wdelim)     # use earliest delim position
+    return url[start:delim], url[delim:]   # return (domain, rest)
+
+def _checknetloc(netloc):
+    if not netloc or netloc.isascii():
+        return
+    # looking for characters like \u2100 that expand to 'a/c'
+    # IDNA uses NFKC equivalence, so normalize for this check
+    import unicodedata
+    n = netloc.replace('@', '')   # ignore characters already included
+    n = n.replace(':', '')        # but not the surrounding text
+    n = n.replace('#', '')
+    n = n.replace('?', '')
+    netloc2 = unicodedata.normalize('NFKC', n)
+    if n == netloc2:
+        return
+    for c in '/?#@:':
+        if c in netloc2:
+            raise ValueError("netloc '" + netloc + "' contains invalid " +
+                             "characters under NFKC normalization")
+
+def _check_bracketed_netloc(netloc):
+    # Note that this function must mirror the splitting
+    # done in NetlocResultMixins._hostinfo().
+    hostname_and_port = netloc.rpartition('@')[2]
+    before_bracket, have_open_br, bracketed = hostname_and_port.partition('[')
+    if have_open_br:
+        # No data is allowed before a bracket.
+        if before_bracket:
+            raise ValueError("Invalid IPv6 URL")
+        hostname, _, port = bracketed.partition(']')
+        # No data is allowed after the bracket but before the port delimiter.
+        if port and not port.startswith(":"):
+            raise ValueError("Invalid IPv6 URL")
+    else:
+        hostname, _, port = hostname_and_port.partition(':')
+    _check_bracketed_host(hostname)
+
+# Valid bracketed hosts are defined in
+# https://www.rfc-editor.org/rfc/rfc3986#page-49 and https://url.spec.whatwg.org/
+def _check_bracketed_host(hostname):
+    if hostname.startswith('v'):
+        if not re.match(r"\Av[a-fA-F0-9]+\..+\Z", hostname):
+            raise ValueError(f"IPvFuture address is invalid")
+    else:
+        ip = ipaddress.ip_address(hostname) # Throws Value Error if not IPv6 or IPv4
+        if isinstance(ip, ipaddress.IPv4Address):
+            raise ValueError(f"An IPv4 address cannot be in brackets")
+
+def urlsplit(url, scheme='', allow_fragments=True):
+    """Parse a URL into 5 components:
+    :///?#
+
+    The result is a named 5-tuple with fields corresponding to the
+    above. It is either a SplitResult or SplitResultBytes object,
+    depending on the type of the url parameter.
+
+    The username, password, hostname, and port sub-components of netloc
+    can also be accessed as attributes of the returned object.
+
+    The scheme argument provides the default value of the scheme
+    component when no scheme is found in url.
+
+    If allow_fragments is False, no attempt is made to separate the
+    fragment component from the previous component, which can be either
+    path or query.
+
+    Note that % escapes are not expanded.
+    """
+
+    url, scheme, _coerce_result = _coerce_args(url, scheme)
+    # Only lstrip url as some applications rely on preserving trailing space.
+    # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both)
+    url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE)
+    scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE)
+
+    for b in _UNSAFE_URL_BYTES_TO_REMOVE:
+        url = url.replace(b, "")
+        scheme = scheme.replace(b, "")
+
+    allow_fragments = bool(allow_fragments)
+    key = url, scheme, allow_fragments, type(url), type(scheme)
+    cached = _parse_cache.get(key, None)
+    if cached:
+        return _coerce_result(cached)
+    if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth
+        clear_cache()
+    netloc = query = fragment = ''
+    i = url.find(':')
+    if i > 0:
+        for c in url[:i]:
+            if c not in scheme_chars:
+                break
+        else:
+            scheme, url = url[:i].lower(), url[i+1:]
+    if url[:2] == '//':
+        netloc, url = _splitnetloc(url, 2)
+        if (('[' in netloc and ']' not in netloc) or
+                (']' in netloc and '[' not in netloc)):
+            raise ValueError("Invalid IPv6 URL")
+        if '[' in netloc and ']' in netloc:
+            _check_bracketed_netloc(netloc)
+    if allow_fragments and '#' in url:
+        url, fragment = url.split('#', 1)
+    if '?' in url:
+        url, query = url.split('?', 1)
+    _checknetloc(netloc)
+    v = SplitResult(scheme, netloc, url, query, fragment)
+    _parse_cache[key] = v
+    return _coerce_result(v)
+
+def urlunparse(components):
+    """Put a parsed URL back together again.  This may result in a
+    slightly different, but equivalent URL, if the URL that was parsed
+    originally had redundant delimiters, e.g. a ? with an empty query
+    (the draft states that these are equivalent)."""
+    scheme, netloc, url, params, query, fragment, _coerce_result = (
+                                                  _coerce_args(*components))
+    if params:
+        url = "%s;%s" % (url, params)
+    return _coerce_result(urlunsplit((scheme, netloc, url, query, fragment)))
+
+def urlunsplit(components):
+    """Combine the elements of a tuple as returned by urlsplit() into a
+    complete URL as a string. The data argument can be any five-item iterable.
+    This may result in a slightly different, but equivalent URL, if the URL that
+    was parsed originally had unnecessary delimiters (for example, a ? with an
+    empty query; the RFC states that these are equivalent)."""
+    scheme, netloc, url, query, fragment, _coerce_result = (
+                                          _coerce_args(*components))
+    if netloc or (scheme and scheme in uses_netloc) or url[:2] == '//':
+        if url and url[:1] != '/': url = '/' + url
+        url = '//' + (netloc or '') + url
+    if scheme:
+        url = scheme + ':' + url
+    if query:
+        url = url + '?' + query
+    if fragment:
+        url = url + '#' + fragment
+    return _coerce_result(url)
+
+def urljoin(base, url, allow_fragments=True):
+    """Join a base URL and a possibly relative URL to form an absolute
+    interpretation of the latter."""
+    if not base:
+        return url
+    if not url:
+        return base
+
+    base, url, _coerce_result = _coerce_args(base, url)
+    bscheme, bnetloc, bpath, bparams, bquery, bfragment = \
+            urlparse(base, '', allow_fragments)
+    scheme, netloc, path, params, query, fragment = \
+            urlparse(url, bscheme, allow_fragments)
+
+    if scheme != bscheme or scheme not in uses_relative:
+        return _coerce_result(url)
+    if scheme in uses_netloc:
+        if netloc:
+            return _coerce_result(urlunparse((scheme, netloc, path,
+                                              params, query, fragment)))
+        netloc = bnetloc
+
+    if not path and not params:
+        path = bpath
+        params = bparams
+        if not query:
+            query = bquery
+        return _coerce_result(urlunparse((scheme, netloc, path,
+                                          params, query, fragment)))
+
+    base_parts = bpath.split('/')
+    if base_parts[-1] != '':
+        # the last item is not a directory, so will not be taken into account
+        # in resolving the relative path
+        del base_parts[-1]
+
+    # for rfc3986, ignore all base path should the first character be root.
+    if path[:1] == '/':
+        segments = path.split('/')
+    else:
+        segments = base_parts + path.split('/')
+        # filter out elements that would cause redundant slashes on re-joining
+        # the resolved_path
+        segments[1:-1] = filter(None, segments[1:-1])
+
+    resolved_path = []
+
+    for seg in segments:
+        if seg == '..':
+            try:
+                resolved_path.pop()
+            except IndexError:
+                # ignore any .. segments that would otherwise cause an IndexError
+                # when popped from resolved_path if resolving for rfc3986
+                pass
+        elif seg == '.':
+            continue
+        else:
+            resolved_path.append(seg)
+
+    if segments[-1] in ('.', '..'):
+        # do some post-processing here. if the last segment was a relative dir,
+        # then we need to append the trailing '/'
+        resolved_path.append('')
+
+    return _coerce_result(urlunparse((scheme, netloc, '/'.join(
+        resolved_path) or '/', params, query, fragment)))
+
+
+def urldefrag(url):
+    """Removes any existing fragment from URL.
+
+    Returns a tuple of the defragmented URL and the fragment.  If
+    the URL contained no fragments, the second element is the
+    empty string.
+    """
+    url, _coerce_result = _coerce_args(url)
+    if '#' in url:
+        s, n, p, a, q, frag = urlparse(url)
+        defrag = urlunparse((s, n, p, a, q, ''))
+    else:
+        frag = ''
+        defrag = url
+    return _coerce_result(DefragResult(defrag, frag))
+
+_hexdig = '0123456789ABCDEFabcdef'
+_hextobyte = None
+
+def unquote_to_bytes(string):
+    """unquote_to_bytes('abc%20def') -> b'abc def'."""
+    # Note: strings are encoded as UTF-8. This is only an issue if it contains
+    # unescaped non-ASCII characters, which URIs should not.
+    if not string:
+        # Is it a string-like object?
+        string.split
+        return b''
+    if isinstance(string, str):
+        string = string.encode('utf-8')
+    bits = string.split(b'%')
+    if len(bits) == 1:
+        return string
+    res = [bits[0]]
+    append = res.append
+    # Delay the initialization of the table to not waste memory
+    # if the function is never called
+    global _hextobyte
+    if _hextobyte is None:
+        _hextobyte = {(a + b).encode(): bytes.fromhex(a + b)
+                      for a in _hexdig for b in _hexdig}
+    for item in bits[1:]:
+        try:
+            append(_hextobyte[item[:2]])
+            append(item[2:])
+        except KeyError:
+            append(b'%')
+            append(item)
+    return b''.join(res)
+
+_asciire = re.compile('([\x00-\x7f]+)')
+
+def unquote(string, encoding='utf-8', errors='replace'):
+    """Replace %xx escapes by their single-character equivalent. The optional
+    encoding and errors parameters specify how to decode percent-encoded
+    sequences into Unicode characters, as accepted by the bytes.decode()
+    method.
+    By default, percent-encoded sequences are decoded with UTF-8, and invalid
+    sequences are replaced by a placeholder character.
+
+    unquote('abc%20def') -> 'abc def'.
+    """
+    if isinstance(string, bytes):
+        return unquote_to_bytes(string).decode(encoding, errors)
+    if '%' not in string:
+        string.split
+        return string
+    if encoding is None:
+        encoding = 'utf-8'
+    if errors is None:
+        errors = 'replace'
+    bits = _asciire.split(string)
+    res = [bits[0]]
+    append = res.append
+    for i in range(1, len(bits), 2):
+        append(unquote_to_bytes(bits[i]).decode(encoding, errors))
+        append(bits[i + 1])
+    return ''.join(res)
+
+
+def parse_qs(qs, keep_blank_values=False, strict_parsing=False,
+             encoding='utf-8', errors='replace', max_num_fields=None, separator='&'):
+    """Parse a query given as a string argument.
+
+        Arguments:
+
+        qs: percent-encoded query string to be parsed
+
+        keep_blank_values: flag indicating whether blank values in
+            percent-encoded queries should be treated as blank strings.
+            A true value indicates that blanks should be retained as
+            blank strings.  The default false value indicates that
+            blank values are to be ignored and treated as if they were
+            not included.
+
+        strict_parsing: flag indicating what to do with parsing errors.
+            If false (the default), errors are silently ignored.
+            If true, errors raise a ValueError exception.
+
+        encoding and errors: specify how to decode percent-encoded sequences
+            into Unicode characters, as accepted by the bytes.decode() method.
+
+        max_num_fields: int. If set, then throws a ValueError if there
+            are more than n fields read by parse_qsl().
+
+        separator: str. The symbol to use for separating the query arguments.
+            Defaults to &.
+
+        Returns a dictionary.
+    """
+    parsed_result = {}
+    pairs = parse_qsl(qs, keep_blank_values, strict_parsing,
+                      encoding=encoding, errors=errors,
+                      max_num_fields=max_num_fields, separator=separator)
+    for name, value in pairs:
+        if name in parsed_result:
+            parsed_result[name].append(value)
+        else:
+            parsed_result[name] = [value]
+    return parsed_result
+
+
+def parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
+              encoding='utf-8', errors='replace', max_num_fields=None, separator='&'):
+    """Parse a query given as a string argument.
+
+        Arguments:
+
+        qs: percent-encoded query string to be parsed
+
+        keep_blank_values: flag indicating whether blank values in
+            percent-encoded queries should be treated as blank strings.
+            A true value indicates that blanks should be retained as blank
+            strings.  The default false value indicates that blank values
+            are to be ignored and treated as if they were  not included.
+
+        strict_parsing: flag indicating what to do with parsing errors. If
+            false (the default), errors are silently ignored. If true,
+            errors raise a ValueError exception.
+
+        encoding and errors: specify how to decode percent-encoded sequences
+            into Unicode characters, as accepted by the bytes.decode() method.
+
+        max_num_fields: int. If set, then throws a ValueError
+            if there are more than n fields read by parse_qsl().
+
+        separator: str. The symbol to use for separating the query arguments.
+            Defaults to &.
+
+        Returns a list, as G-d intended.
+    """
+    qs, _coerce_result = _coerce_args(qs)
+    separator, _ = _coerce_args(separator)
+
+    if not separator or (not isinstance(separator, (str, bytes))):
+        raise ValueError("Separator must be of type string or bytes.")
+
+    # If max_num_fields is defined then check that the number of fields
+    # is less than max_num_fields. This prevents a memory exhaustion DOS
+    # attack via post bodies with many fields.
+    if max_num_fields is not None:
+        num_fields = 1 + qs.count(separator)
+        if max_num_fields < num_fields:
+            raise ValueError('Max number of fields exceeded')
+
+    r = []
+    for name_value in qs.split(separator):
+        if not name_value and not strict_parsing:
+            continue
+        nv = name_value.split('=', 1)
+        if len(nv) != 2:
+            if strict_parsing:
+                raise ValueError("bad query field: %r" % (name_value,))
+            # Handle case of a control-name with no equal sign
+            if keep_blank_values:
+                nv.append('')
+            else:
+                continue
+        if len(nv[1]) or keep_blank_values:
+            name = nv[0].replace('+', ' ')
+            name = unquote(name, encoding=encoding, errors=errors)
+            name = _coerce_result(name)
+            value = nv[1].replace('+', ' ')
+            value = unquote(value, encoding=encoding, errors=errors)
+            value = _coerce_result(value)
+            r.append((name, value))
+    return r
+
+def unquote_plus(string, encoding='utf-8', errors='replace'):
+    """Like unquote(), but also replace plus signs by spaces, as required for
+    unquoting HTML form values.
+
+    unquote_plus('%7e/abc+def') -> '~/abc def'
+    """
+    string = string.replace('+', ' ')
+    return unquote(string, encoding, errors)
+
+_ALWAYS_SAFE = frozenset(b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+                         b'abcdefghijklmnopqrstuvwxyz'
+                         b'0123456789'
+                         b'_.-~')
+_ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE)
+_safe_quoters = {}
+
+class Quoter(collections.defaultdict):
+    """A mapping from bytes (in range(0,256)) to strings.
+
+    String values are percent-encoded byte values, unless the key < 128, and
+    in the "safe" set (either the specified safe set, or default set).
+    """
+    # Keeps a cache internally, using defaultdict, for efficiency (lookups
+    # of cached keys don't call Python code at all).
+    def __init__(self, safe):
+        """safe: bytes object."""
+        self.safe = _ALWAYS_SAFE.union(safe)
+
+    def __repr__(self):
+        # Without this, will just display as a defaultdict
+        return "<%s %r>" % (self.__class__.__name__, dict(self))
+
+    def __missing__(self, b):
+        # Handle a cache miss. Store quoted string in cache and return.
+        res = chr(b) if b in self.safe else '%{:02X}'.format(b)
+        self[b] = res
+        return res
+
+def quote(string, safe='/', encoding=None, errors=None):
+    """quote('abc def') -> 'abc%20def'
+
+    Each part of a URL, e.g. the path info, the query, etc., has a
+    different set of reserved characters that must be quoted. The
+    quote function offers a cautious (not minimal) way to quote a
+    string for most of these parts.
+
+    RFC 3986 Uniform Resource Identifier (URI): Generic Syntax lists
+    the following (un)reserved characters.
+
+    unreserved    = ALPHA / DIGIT / "-" / "." / "_" / "~"
+    reserved      = gen-delims / sub-delims
+    gen-delims    = ":" / "/" / "?" / "#" / "[" / "]" / "@"
+    sub-delims    = "!" / "$" / "&" / "'" / "(" / ")"
+                  / "*" / "+" / "," / ";" / "="
+
+    Each of the reserved characters is reserved in some component of a URL,
+    but not necessarily in all of them.
+
+    The quote function %-escapes all characters that are neither in the
+    unreserved chars ("always safe") nor the additional chars set via the
+    safe arg.
+
+    The default for the safe arg is '/'. The character is reserved, but in
+    typical usage the quote function is being called on a path where the
+    existing slash characters are to be preserved.
+
+    Python 3.7 updates from using RFC 2396 to RFC 3986 to quote URL strings.
+    Now, "~" is included in the set of unreserved characters.
+
+    string and safe may be either str or bytes objects. encoding and errors
+    must not be specified if string is a bytes object.
+
+    The optional encoding and errors parameters specify how to deal with
+    non-ASCII characters, as accepted by the str.encode method.
+    By default, encoding='utf-8' (characters are encoded with UTF-8), and
+    errors='strict' (unsupported characters raise a UnicodeEncodeError).
+    """
+    if isinstance(string, str):
+        if not string:
+            return string
+        if encoding is None:
+            encoding = 'utf-8'
+        if errors is None:
+            errors = 'strict'
+        string = string.encode(encoding, errors)
+    else:
+        if encoding is not None:
+            raise TypeError("quote() doesn't support 'encoding' for bytes")
+        if errors is not None:
+            raise TypeError("quote() doesn't support 'errors' for bytes")
+    return quote_from_bytes(string, safe)
+
+def quote_plus(string, safe='', encoding=None, errors=None):
+    """Like quote(), but also replace ' ' with '+', as required for quoting
+    HTML form values. Plus signs in the original string are escaped unless
+    they are included in safe. It also does not have safe default to '/'.
+    """
+    # Check if ' ' in string, where string may either be a str or bytes.  If
+    # there are no spaces, the regular quote will produce the right answer.
+    if ((isinstance(string, str) and ' ' not in string) or
+        (isinstance(string, bytes) and b' ' not in string)):
+        return quote(string, safe, encoding, errors)
+    if isinstance(safe, str):
+        space = ' '
+    else:
+        space = b' '
+    string = quote(string, safe + space, encoding, errors)
+    return string.replace(' ', '+')
+
+def quote_from_bytes(bs, safe='/'):
+    """Like quote(), but accepts a bytes object rather than a str, and does
+    not perform string-to-bytes encoding.  It always returns an ASCII string.
+    quote_from_bytes(b'abc def\x3f') -> 'abc%20def%3f'
+    """
+    if not isinstance(bs, (bytes, bytearray)):
+        raise TypeError("quote_from_bytes() expected bytes")
+    if not bs:
+        return ''
+    if isinstance(safe, str):
+        # Normalize 'safe' by converting to bytes and removing non-ASCII chars
+        safe = safe.encode('ascii', 'ignore')
+    else:
+        safe = bytes([c for c in safe if c < 128])
+    if not bs.rstrip(_ALWAYS_SAFE_BYTES + safe):
+        return bs.decode()
+    try:
+        quoter = _safe_quoters[safe]
+    except KeyError:
+        _safe_quoters[safe] = quoter = Quoter(safe).__getitem__
+    return ''.join([quoter(char) for char in bs])
+
+def urlencode(query, doseq=False, safe='', encoding=None, errors=None,
+              quote_via=quote_plus):
+    """Encode a dict or sequence of two-element tuples into a URL query string.
+
+    If any values in the query arg are sequences and doseq is true, each
+    sequence element is converted to a separate parameter.
+
+    If the query arg is a sequence of two-element tuples, the order of the
+    parameters in the output will match the order of parameters in the
+    input.
+
+    The components of a query arg may each be either a string or a bytes type.
+
+    The safe, encoding, and errors parameters are passed down to the function
+    specified by quote_via (encoding and errors only if a component is a str).
+    """
+
+    if hasattr(query, "items"):
+        query = query.items()
+    else:
+        # It's a bother at times that strings and string-like objects are
+        # sequences.
+        try:
+            # non-sequence items should not work with len()
+            # non-empty strings will fail this
+            if len(query) and not isinstance(query[0], tuple):
+                raise TypeError
+            # Zero-length sequences of all types will get here and succeed,
+            # but that's a minor nit.  Since the original implementation
+            # allowed empty dicts that type of behavior probably should be
+            # preserved for consistency
+        except TypeError:
+            ty, va, tb = sys.exc_info()
+            raise TypeError("not a valid non-string sequence "
+                            "or mapping object").with_traceback(tb)
+
+    l = []
+    if not doseq:
+        for k, v in query:
+            if isinstance(k, bytes):
+                k = quote_via(k, safe)
+            else:
+                k = quote_via(str(k), safe, encoding, errors)
+
+            if isinstance(v, bytes):
+                v = quote_via(v, safe)
+            else:
+                v = quote_via(str(v), safe, encoding, errors)
+            l.append(k + '=' + v)
+    else:
+        for k, v in query:
+            if isinstance(k, bytes):
+                k = quote_via(k, safe)
+            else:
+                k = quote_via(str(k), safe, encoding, errors)
+
+            if isinstance(v, bytes):
+                v = quote_via(v, safe)
+                l.append(k + '=' + v)
+            elif isinstance(v, str):
+                v = quote_via(v, safe, encoding, errors)
+                l.append(k + '=' + v)
+            else:
+                try:
+                    # Is this a sufficient test for sequence-ness?
+                    x = len(v)
+                except TypeError:
+                    # not a sequence
+                    v = quote_via(str(v), safe, encoding, errors)
+                    l.append(k + '=' + v)
+                else:
+                    # loop over the sequence
+                    for elt in v:
+                        if isinstance(elt, bytes):
+                            elt = quote_via(elt, safe)
+                        else:
+                            elt = quote_via(str(elt), safe, encoding, errors)
+                        l.append(k + '=' + elt)
+    return '&'.join(l)
+
+
+def to_bytes(url):
+    warnings.warn("urllib.parse.to_bytes() is deprecated as of 3.8",
+                  DeprecationWarning, stacklevel=2)
+    return _to_bytes(url)
+
+
+def _to_bytes(url):
+    """to_bytes(u"URL") --> 'URL'."""
+    # Most URL schemes require ASCII. If that changes, the conversion
+    # can be relaxed.
+    # XXX get rid of to_bytes()
+    if isinstance(url, str):
+        try:
+            url = url.encode("ASCII").decode()
+        except UnicodeError:
+            raise UnicodeError("URL " + repr(url) +
+                               " contains non-ASCII characters")
+    return url
+
+
+def unwrap(url):
+    """Transform a string like '' into 'scheme://host/path'.
+
+    The string is returned unchanged if it's not a wrapped URL.
+    """
+    url = str(url).strip()
+    if url[:1] == '<' and url[-1:] == '>':
+        url = url[1:-1].strip()
+    if url[:4] == 'URL:':
+        url = url[4:].strip()
+    return url
+
+
+def splittype(url):
+    warnings.warn("urllib.parse.splittype() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splittype(url)
+
+
+_typeprog = None
+def _splittype(url):
+    """splittype('type:opaquestring') --> 'type', 'opaquestring'."""
+    global _typeprog
+    if _typeprog is None:
+        _typeprog = re.compile('([^/:]+):(.*)', re.DOTALL)
+
+    match = _typeprog.match(url)
+    if match:
+        scheme, data = match.groups()
+        return scheme.lower(), data
+    return None, url
+
+
+def splithost(url):
+    warnings.warn("urllib.parse.splithost() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splithost(url)
+
+
+_hostprog = None
+def _splithost(url):
+    """splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
+    global _hostprog
+    if _hostprog is None:
+        _hostprog = re.compile('//([^/#?]*)(.*)', re.DOTALL)
+
+    match = _hostprog.match(url)
+    if match:
+        host_port, path = match.groups()
+        if path and path[0] != '/':
+            path = '/' + path
+        return host_port, path
+    return None, url
+
+
+def splituser(host):
+    warnings.warn("urllib.parse.splituser() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splituser(host)
+
+
+def _splituser(host):
+    """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
+    user, delim, host = host.rpartition('@')
+    return (user if delim else None), host
+
+
+def splitpasswd(user):
+    warnings.warn("urllib.parse.splitpasswd() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splitpasswd(user)
+
+
+def _splitpasswd(user):
+    """splitpasswd('user:passwd') -> 'user', 'passwd'."""
+    user, delim, passwd = user.partition(':')
+    return user, (passwd if delim else None)
+
+
+def splitport(host):
+    warnings.warn("urllib.parse.splitport() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splitport(host)
+
+
+# splittag('/path#tag') --> '/path', 'tag'
+_portprog = None
+def _splitport(host):
+    """splitport('host:port') --> 'host', 'port'."""
+    global _portprog
+    if _portprog is None:
+        _portprog = re.compile('(.*):([0-9]*)', re.DOTALL)
+
+    match = _portprog.fullmatch(host)
+    if match:
+        host, port = match.groups()
+        if port:
+            return host, port
+    return host, None
+
+
+def splitnport(host, defport=-1):
+    warnings.warn("urllib.parse.splitnport() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splitnport(host, defport)
+
+
+def _splitnport(host, defport=-1):
+    """Split host and port, returning numeric port.
+    Return given default port if no ':' found; defaults to -1.
+    Return numerical port if a valid number is found after ':'.
+    Return None if ':' but not a valid number."""
+    host, delim, port = host.rpartition(':')
+    if not delim:
+        host = port
+    elif port:
+        if port.isdigit() and port.isascii():
+            nport = int(port)
+        else:
+            nport = None
+        return host, nport
+    return host, defport
+
+
+def splitquery(url):
+    warnings.warn("urllib.parse.splitquery() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splitquery(url)
+
+
+def _splitquery(url):
+    """splitquery('/path?query') --> '/path', 'query'."""
+    path, delim, query = url.rpartition('?')
+    if delim:
+        return path, query
+    return url, None
+
+
+def splittag(url):
+    warnings.warn("urllib.parse.splittag() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splittag(url)
+
+
+def _splittag(url):
+    """splittag('/path#tag') --> '/path', 'tag'."""
+    path, delim, tag = url.rpartition('#')
+    if delim:
+        return path, tag
+    return url, None
+
+
+def splitattr(url):
+    warnings.warn("urllib.parse.splitattr() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splitattr(url)
+
+
+def _splitattr(url):
+    """splitattr('/path;attr1=value1;attr2=value2;...') ->
+        '/path', ['attr1=value1', 'attr2=value2', ...]."""
+    words = url.split(';')
+    return words[0], words[1:]
+
+
+def splitvalue(attr):
+    warnings.warn("urllib.parse.splitvalue() is deprecated as of 3.8, "
+                  "use urllib.parse.parse_qsl() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splitvalue(attr)
+
+
+def _splitvalue(attr):
+    """splitvalue('attr=value') --> 'attr', 'value'."""
+    attr, delim, value = attr.partition('=')
+    return attr, (value if delim else None)
diff --git a/lib/python3.1/urllib/request.py b/lib/python3.1/urllib/request.py
new file mode 100644
index 0000000000000000000000000000000000000000..6edde1f73189b1d2fa38c8b714c5b62864c21aef
--- /dev/null
+++ b/lib/python3.1/urllib/request.py
@@ -0,0 +1,2778 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below).  It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work.  Each Handler implements a particular protocol or
+option.  The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL.  For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns.  The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib.  pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back.  One difference is that you can also pass
+a Request instance instead of URL.  Raises a URLError (subclass of
+OSError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers.  Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate.  If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request.  The
+state can be as simple as the URL.  It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib.request
+
+# set up authentication info
+authinfo = urllib.request.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+                      uri='https://mahler:8092/site-updates.py',
+                      user='klem',
+                      passwd='geheim$parole')
+
+proxy_support = urllib.request.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib.request.build_opener(proxy_support, authinfo,
+                                     urllib.request.CacheFTPHandler)
+
+# install it
+urllib.request.install_opener(opener)
+
+f = urllib.request.urlopen('https://www.python.org/')
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled?  The client needs to know the HTTP error code.  But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies  XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import bisect
+import email
+import hashlib
+import http.client
+import io
+import os
+import posixpath
+import re
+import socket
+import string
+import sys
+import time
+import tempfile
+import contextlib
+import warnings
+
+
+from urllib.error import URLError, HTTPError, ContentTooShortError
+from urllib.parse import (
+    urlparse, urlsplit, urljoin, unwrap, quote, unquote,
+    _splittype, _splithost, _splitport, _splituser, _splitpasswd,
+    _splitattr, _splitquery, _splitvalue, _splittag, _to_bytes,
+    unquote_to_bytes, urlunparse)
+from urllib.response import addinfourl, addclosehook
+
+# check for SSL
+try:
+    import ssl
+except ImportError:
+    _have_ssl = False
+else:
+    _have_ssl = True
+
+__all__ = [
+    # Classes
+    'Request', 'OpenerDirector', 'BaseHandler', 'HTTPDefaultErrorHandler',
+    'HTTPRedirectHandler', 'HTTPCookieProcessor', 'ProxyHandler',
+    'HTTPPasswordMgr', 'HTTPPasswordMgrWithDefaultRealm',
+    'HTTPPasswordMgrWithPriorAuth', 'AbstractBasicAuthHandler',
+    'HTTPBasicAuthHandler', 'ProxyBasicAuthHandler', 'AbstractDigestAuthHandler',
+    'HTTPDigestAuthHandler', 'ProxyDigestAuthHandler', 'HTTPHandler',
+    'FileHandler', 'FTPHandler', 'CacheFTPHandler', 'DataHandler',
+    'UnknownHandler', 'HTTPErrorProcessor',
+    # Functions
+    'urlopen', 'install_opener', 'build_opener',
+    'pathname2url', 'url2pathname', 'getproxies',
+    # Legacy interface
+    'urlretrieve', 'urlcleanup', 'URLopener', 'FancyURLopener',
+]
+
+# used in User-Agent header sent
+__version__ = '%d.%d' % sys.version_info[:2]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+            *, cafile=None, capath=None, cadefault=False, context=None):
+    '''Open the URL url, which can be either a string or a Request object.
+
+    *data* must be an object specifying additional data to be sent to
+    the server, or None if no such data is needed.  See Request for
+    details.
+
+    urllib.request module uses HTTP/1.1 and includes a "Connection:close"
+    header in its HTTP requests.
+
+    The optional *timeout* parameter specifies a timeout in seconds for
+    blocking operations like the connection attempt (if not specified, the
+    global default timeout setting will be used). This only works for HTTP,
+    HTTPS and FTP connections.
+
+    If *context* is specified, it must be a ssl.SSLContext instance describing
+    the various SSL options. See HTTPSConnection for more details.
+
+    The optional *cafile* and *capath* parameters specify a set of trusted CA
+    certificates for HTTPS requests. cafile should point to a single file
+    containing a bundle of CA certificates, whereas capath should point to a
+    directory of hashed certificate files. More information can be found in
+    ssl.SSLContext.load_verify_locations().
+
+    The *cadefault* parameter is ignored.
+
+
+    This function always returns an object which can work as a
+    context manager and has the properties url, headers, and status.
+    See urllib.response.addinfourl for more detail on these properties.
+
+    For HTTP and HTTPS URLs, this function returns a http.client.HTTPResponse
+    object slightly modified. In addition to the three new methods above, the
+    msg attribute contains the same information as the reason attribute ---
+    the reason phrase returned by the server --- instead of the response
+    headers as it is specified in the documentation for HTTPResponse.
+
+    For FTP, file, and data URLs and requests explicitly handled by legacy
+    URLopener and FancyURLopener classes, this function returns a
+    urllib.response.addinfourl object.
+
+    Note that None may be returned if no handler handles the request (though
+    the default installed global OpenerDirector uses UnknownHandler to ensure
+    this never happens).
+
+    In addition, if proxy settings are detected (for example, when a *_proxy
+    environment variable like http_proxy is set), ProxyHandler is default
+    installed and makes sure the requests are handled through the proxy.
+
+    '''
+    global _opener
+    if cafile or capath or cadefault:
+        import warnings
+        warnings.warn("cafile, capath and cadefault are deprecated, use a "
+                      "custom context instead.", DeprecationWarning, 2)
+        if context is not None:
+            raise ValueError(
+                "You can't pass both context and any of cafile, capath, and "
+                "cadefault"
+            )
+        if not _have_ssl:
+            raise ValueError('SSL support not available')
+        context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH,
+                                             cafile=cafile,
+                                             capath=capath)
+        # send ALPN extension to indicate HTTP/1.1 protocol
+        context.set_alpn_protocols(['http/1.1'])
+        https_handler = HTTPSHandler(context=context)
+        opener = build_opener(https_handler)
+    elif context:
+        https_handler = HTTPSHandler(context=context)
+        opener = build_opener(https_handler)
+    elif _opener is None:
+        _opener = opener = build_opener()
+    else:
+        opener = _opener
+    return opener.open(url, data, timeout)
+
+def install_opener(opener):
+    global _opener
+    _opener = opener
+
+_url_tempfiles = []
+def urlretrieve(url, filename=None, reporthook=None, data=None):
+    """
+    Retrieve a URL into a temporary location on disk.
+
+    Requires a URL argument. If a filename is passed, it is used as
+    the temporary file location. The reporthook argument should be
+    a callable that accepts a block number, a read size, and the
+    total file size of the URL target. The data argument should be
+    valid URL encoded data.
+
+    If a filename is passed and the URL points to a local resource,
+    the result is a copy from local file to new file.
+
+    Returns a tuple containing the path to the newly created
+    data file as well as the resulting HTTPMessage object.
+    """
+    url_type, path = _splittype(url)
+
+    with contextlib.closing(urlopen(url, data)) as fp:
+        headers = fp.info()
+
+        # Just return the local path and the "headers" for file://
+        # URLs. No sense in performing a copy unless requested.
+        if url_type == "file" and not filename:
+            return os.path.normpath(path), headers
+
+        # Handle temporary file setup.
+        if filename:
+            tfp = open(filename, 'wb')
+        else:
+            tfp = tempfile.NamedTemporaryFile(delete=False)
+            filename = tfp.name
+            _url_tempfiles.append(filename)
+
+        with tfp:
+            result = filename, headers
+            bs = 1024*8
+            size = -1
+            read = 0
+            blocknum = 0
+            if "content-length" in headers:
+                size = int(headers["Content-Length"])
+
+            if reporthook:
+                reporthook(blocknum, bs, size)
+
+            while True:
+                block = fp.read(bs)
+                if not block:
+                    break
+                read += len(block)
+                tfp.write(block)
+                blocknum += 1
+                if reporthook:
+                    reporthook(blocknum, bs, size)
+
+    if size >= 0 and read < size:
+        raise ContentTooShortError(
+            "retrieval incomplete: got only %i out of %i bytes"
+            % (read, size), result)
+
+    return result
+
+def urlcleanup():
+    """Clean up temporary files from urlretrieve calls."""
+    for temp_file in _url_tempfiles:
+        try:
+            os.unlink(temp_file)
+        except OSError:
+            pass
+
+    del _url_tempfiles[:]
+    global _opener
+    if _opener:
+        _opener = None
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$", re.ASCII)
+def request_host(request):
+    """Return request-host, as defined by RFC 2965.
+
+    Variation from RFC: returned value is lowercased, for convenient
+    comparison.
+
+    """
+    url = request.full_url
+    host = urlparse(url)[1]
+    if host == "":
+        host = request.get_header("Host", "")
+
+    # remove port, if present
+    host = _cut_port_re.sub("", host, 1)
+    return host.lower()
+
+class Request:
+
+    def __init__(self, url, data=None, headers={},
+                 origin_req_host=None, unverifiable=False,
+                 method=None):
+        self.full_url = url
+        self.headers = {}
+        self.unredirected_hdrs = {}
+        self._data = None
+        self.data = data
+        self._tunnel_host = None
+        for key, value in headers.items():
+            self.add_header(key, value)
+        if origin_req_host is None:
+            origin_req_host = request_host(self)
+        self.origin_req_host = origin_req_host
+        self.unverifiable = unverifiable
+        if method:
+            self.method = method
+
+    @property
+    def full_url(self):
+        if self.fragment:
+            return '{}#{}'.format(self._full_url, self.fragment)
+        return self._full_url
+
+    @full_url.setter
+    def full_url(self, url):
+        # unwrap('') --> 'type://host/path'
+        self._full_url = unwrap(url)
+        self._full_url, self.fragment = _splittag(self._full_url)
+        self._parse()
+
+    @full_url.deleter
+    def full_url(self):
+        self._full_url = None
+        self.fragment = None
+        self.selector = ''
+
+    @property
+    def data(self):
+        return self._data
+
+    @data.setter
+    def data(self, data):
+        if data != self._data:
+            self._data = data
+            # issue 16464
+            # if we change data we need to remove content-length header
+            # (cause it's most probably calculated for previous value)
+            if self.has_header("Content-length"):
+                self.remove_header("Content-length")
+
+    @data.deleter
+    def data(self):
+        self.data = None
+
+    def _parse(self):
+        self.type, rest = _splittype(self._full_url)
+        if self.type is None:
+            raise ValueError("unknown url type: %r" % self.full_url)
+        self.host, self.selector = _splithost(rest)
+        if self.host:
+            self.host = unquote(self.host)
+
+    def get_method(self):
+        """Return a string indicating the HTTP request method."""
+        default_method = "POST" if self.data is not None else "GET"
+        return getattr(self, 'method', default_method)
+
+    def get_full_url(self):
+        return self.full_url
+
+    def set_proxy(self, host, type):
+        if self.type == 'https' and not self._tunnel_host:
+            self._tunnel_host = self.host
+        else:
+            self.type= type
+            self.selector = self.full_url
+        self.host = host
+
+    def has_proxy(self):
+        return self.selector == self.full_url
+
+    def add_header(self, key, val):
+        # useful for something like authentication
+        self.headers[key.capitalize()] = val
+
+    def add_unredirected_header(self, key, val):
+        # will not be added to a redirected request
+        self.unredirected_hdrs[key.capitalize()] = val
+
+    def has_header(self, header_name):
+        return (header_name in self.headers or
+                header_name in self.unredirected_hdrs)
+
+    def get_header(self, header_name, default=None):
+        return self.headers.get(
+            header_name,
+            self.unredirected_hdrs.get(header_name, default))
+
+    def remove_header(self, header_name):
+        self.headers.pop(header_name, None)
+        self.unredirected_hdrs.pop(header_name, None)
+
+    def header_items(self):
+        hdrs = {**self.unredirected_hdrs, **self.headers}
+        return list(hdrs.items())
+
+class OpenerDirector:
+    def __init__(self):
+        client_version = "Python-urllib/%s" % __version__
+        self.addheaders = [('User-agent', client_version)]
+        # self.handlers is retained only for backward compatibility
+        self.handlers = []
+        # manage the individual handlers
+        self.handle_open = {}
+        self.handle_error = {}
+        self.process_response = {}
+        self.process_request = {}
+
+    def add_handler(self, handler):
+        if not hasattr(handler, "add_parent"):
+            raise TypeError("expected BaseHandler instance, got %r" %
+                            type(handler))
+
+        added = False
+        for meth in dir(handler):
+            if meth in ["redirect_request", "do_open", "proxy_open"]:
+                # oops, coincidental match
+                continue
+
+            i = meth.find("_")
+            protocol = meth[:i]
+            condition = meth[i+1:]
+
+            if condition.startswith("error"):
+                j = condition.find("_") + i + 1
+                kind = meth[j+1:]
+                try:
+                    kind = int(kind)
+                except ValueError:
+                    pass
+                lookup = self.handle_error.get(protocol, {})
+                self.handle_error[protocol] = lookup
+            elif condition == "open":
+                kind = protocol
+                lookup = self.handle_open
+            elif condition == "response":
+                kind = protocol
+                lookup = self.process_response
+            elif condition == "request":
+                kind = protocol
+                lookup = self.process_request
+            else:
+                continue
+
+            handlers = lookup.setdefault(kind, [])
+            if handlers:
+                bisect.insort(handlers, handler)
+            else:
+                handlers.append(handler)
+            added = True
+
+        if added:
+            bisect.insort(self.handlers, handler)
+            handler.add_parent(self)
+
+    def close(self):
+        # Only exists for backwards compatibility.
+        pass
+
+    def _call_chain(self, chain, kind, meth_name, *args):
+        # Handlers raise an exception if no one else should try to handle
+        # the request, or return None if they can't but another handler
+        # could.  Otherwise, they return the response.
+        handlers = chain.get(kind, ())
+        for handler in handlers:
+            func = getattr(handler, meth_name)
+            result = func(*args)
+            if result is not None:
+                return result
+
+    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+        # accept a URL or a Request object
+        if isinstance(fullurl, str):
+            req = Request(fullurl, data)
+        else:
+            req = fullurl
+            if data is not None:
+                req.data = data
+
+        req.timeout = timeout
+        protocol = req.type
+
+        # pre-process request
+        meth_name = protocol+"_request"
+        for processor in self.process_request.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            req = meth(req)
+
+        sys.audit('urllib.Request', req.full_url, req.data, req.headers, req.get_method())
+        response = self._open(req, data)
+
+        # post-process response
+        meth_name = protocol+"_response"
+        for processor in self.process_response.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            response = meth(req, response)
+
+        return response
+
+    def _open(self, req, data=None):
+        result = self._call_chain(self.handle_open, 'default',
+                                  'default_open', req)
+        if result:
+            return result
+
+        protocol = req.type
+        result = self._call_chain(self.handle_open, protocol, protocol +
+                                  '_open', req)
+        if result:
+            return result
+
+        return self._call_chain(self.handle_open, 'unknown',
+                                'unknown_open', req)
+
+    def error(self, proto, *args):
+        if proto in ('http', 'https'):
+            # XXX http[s] protocols are special-cased
+            dict = self.handle_error['http'] # https is not different than http
+            proto = args[2]  # YUCK!
+            meth_name = 'http_error_%s' % proto
+            http_err = 1
+            orig_args = args
+        else:
+            dict = self.handle_error
+            meth_name = proto + '_error'
+            http_err = 0
+        args = (dict, proto, meth_name) + args
+        result = self._call_chain(*args)
+        if result:
+            return result
+
+        if http_err:
+            args = (dict, 'default', 'http_error_default') + orig_args
+            return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+    """Create an opener object from a list of handlers.
+
+    The opener will use several default handlers, including support
+    for HTTP, FTP and when applicable HTTPS.
+
+    If any of the handlers passed as arguments are subclasses of the
+    default handlers, the default handlers will not be used.
+    """
+    opener = OpenerDirector()
+    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
+                       FTPHandler, FileHandler, HTTPErrorProcessor,
+                       DataHandler]
+    if hasattr(http.client, "HTTPSConnection"):
+        default_classes.append(HTTPSHandler)
+    skip = set()
+    for klass in default_classes:
+        for check in handlers:
+            if isinstance(check, type):
+                if issubclass(check, klass):
+                    skip.add(klass)
+            elif isinstance(check, klass):
+                skip.add(klass)
+    for klass in skip:
+        default_classes.remove(klass)
+
+    for klass in default_classes:
+        opener.add_handler(klass())
+
+    for h in handlers:
+        if isinstance(h, type):
+            h = h()
+        opener.add_handler(h)
+    return opener
+
+class BaseHandler:
+    handler_order = 500
+
+    def add_parent(self, parent):
+        self.parent = parent
+
+    def close(self):
+        # Only exists for backwards compatibility
+        pass
+
+    def __lt__(self, other):
+        if not hasattr(other, "handler_order"):
+            # Try to preserve the old behavior of having custom classes
+            # inserted after default ones (works only for custom user
+            # classes which are not aware of handler_order).
+            return True
+        return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+    """Process HTTP error responses."""
+    handler_order = 1000  # after all other processing
+
+    def http_response(self, request, response):
+        code, msg, hdrs = response.code, response.msg, response.info()
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if not (200 <= code < 300):
+            response = self.parent.error(
+                'http', request, response, code, msg, hdrs)
+
+        return response
+
+    https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+    def http_error_default(self, req, fp, code, msg, hdrs):
+        raise HTTPError(req.full_url, code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+    # maximum number of redirections to any single URL
+    # this is needed because of the state that cookies introduce
+    max_repeats = 4
+    # maximum total number of redirections (regardless of URL) before
+    # assuming we're in a loop
+    max_redirections = 10
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        """Return a Request or None in response to a redirect.
+
+        This is called by the http_error_30x methods when a
+        redirection response is received.  If a redirection should
+        take place, return a new Request to allow http_error_30x to
+        perform the redirect.  Otherwise, raise HTTPError if no-one
+        else should try to handle this url.  Return None if you can't
+        but another Handler might.
+        """
+        m = req.get_method()
+        if (not (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+            or code in (301, 302, 303) and m == "POST")):
+            raise HTTPError(req.full_url, code, msg, headers, fp)
+
+        # Strictly (according to RFC 2616), 301 or 302 in response to
+        # a POST MUST NOT cause a redirection without confirmation
+        # from the user (of urllib.request, in this case).  In practice,
+        # essentially all clients do redirect in this case, so we do
+        # the same.
+
+        # Be conciliant with URIs containing a space.  This is mainly
+        # redundant with the more complete encoding done in http_error_302(),
+        # but it is kept for compatibility with other callers.
+        newurl = newurl.replace(' ', '%20')
+
+        CONTENT_HEADERS = ("content-length", "content-type")
+        newheaders = {k: v for k, v in req.headers.items()
+                      if k.lower() not in CONTENT_HEADERS}
+        return Request(newurl,
+                       headers=newheaders,
+                       origin_req_host=req.origin_req_host,
+                       unverifiable=True)
+
+    # Implementation note: To avoid the server sending us into an
+    # infinite loop, the request object needs to track what URLs we
+    # have already seen.  Do this by adding a handler-specific
+    # attribute to the Request object.
+    def http_error_302(self, req, fp, code, msg, headers):
+        # Some servers (incorrectly) return multiple Location headers
+        # (so probably same goes for URI).  Use first header.
+        if "location" in headers:
+            newurl = headers["location"]
+        elif "uri" in headers:
+            newurl = headers["uri"]
+        else:
+            return
+
+        # fix a possible malformed URL
+        urlparts = urlparse(newurl)
+
+        # For security reasons we don't allow redirection to anything other
+        # than http, https or ftp.
+
+        if urlparts.scheme not in ('http', 'https', 'ftp', ''):
+            raise HTTPError(
+                newurl, code,
+                "%s - Redirection to url '%s' is not allowed" % (msg, newurl),
+                headers, fp)
+
+        if not urlparts.path and urlparts.netloc:
+            urlparts = list(urlparts)
+            urlparts[2] = "/"
+        newurl = urlunparse(urlparts)
+
+        # http.client.parse_headers() decodes as ISO-8859-1.  Recover the
+        # original bytes and percent-encode non-ASCII bytes, and any special
+        # characters such as the space.
+        newurl = quote(
+            newurl, encoding="iso-8859-1", safe=string.punctuation)
+        newurl = urljoin(req.full_url, newurl)
+
+        # XXX Probably want to forget about the state of the current
+        # request, although that might interact poorly with other
+        # handlers that also use handler-specific request attributes
+        new = self.redirect_request(req, fp, code, msg, headers, newurl)
+        if new is None:
+            return
+
+        # loop detection
+        # .redirect_dict has a key url if url was previously visited.
+        if hasattr(req, 'redirect_dict'):
+            visited = new.redirect_dict = req.redirect_dict
+            if (visited.get(newurl, 0) >= self.max_repeats or
+                len(visited) >= self.max_redirections):
+                raise HTTPError(req.full_url, code,
+                                self.inf_msg + msg, headers, fp)
+        else:
+            visited = new.redirect_dict = req.redirect_dict = {}
+        visited[newurl] = visited.get(newurl, 0) + 1
+
+        # Don't close the fp until we are sure that we won't use it
+        # with HTTPError.
+        fp.read()
+        fp.close()
+
+        return self.parent.open(new, timeout=req.timeout)
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+    inf_msg = "The HTTP server returned a redirect error that would " \
+              "lead to an infinite loop.\n" \
+              "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+    """Return (scheme, user, password, host/port) given a URL or an authority.
+
+    If a URL is supplied, it must have an authority (host:port) component.
+    According to RFC 3986, having an authority component means the URL must
+    have two slashes after the scheme.
+    """
+    scheme, r_scheme = _splittype(proxy)
+    if not r_scheme.startswith("/"):
+        # authority
+        scheme = None
+        authority = proxy
+    else:
+        # URL
+        if not r_scheme.startswith("//"):
+            raise ValueError("proxy URL with no authority: %r" % proxy)
+        # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+        # and 3.3.), path is empty or starts with '/'
+        if '@' in r_scheme:
+            host_separator = r_scheme.find('@')
+            end = r_scheme.find("/", host_separator)
+        else:
+            end = r_scheme.find("/", 2)
+        if end == -1:
+            end = None
+        authority = r_scheme[2:end]
+    userinfo, hostport = _splituser(authority)
+    if userinfo is not None:
+        user, password = _splitpasswd(userinfo)
+    else:
+        user = password = None
+    return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+    # Proxies must be in front
+    handler_order = 100
+
+    def __init__(self, proxies=None):
+        if proxies is None:
+            proxies = getproxies()
+        assert hasattr(proxies, 'keys'), "proxies must be a mapping"
+        self.proxies = proxies
+        for type, url in proxies.items():
+            type = type.lower()
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy=url, type=type, meth=self.proxy_open:
+                        meth(r, proxy, type))
+
+    def proxy_open(self, req, proxy, type):
+        orig_type = req.type
+        proxy_type, user, password, hostport = _parse_proxy(proxy)
+        if proxy_type is None:
+            proxy_type = orig_type
+
+        if req.host and proxy_bypass(req.host):
+            return None
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user),
+                                   unquote(password))
+            creds = base64.b64encode(user_pass.encode()).decode("ascii")
+            req.add_header('Proxy-authorization', 'Basic ' + creds)
+        hostport = unquote(hostport)
+        req.set_proxy(hostport, proxy_type)
+        if orig_type == proxy_type or orig_type == 'https':
+            # let other handlers take care of it
+            return None
+        else:
+            # need to start over, because the other handlers don't
+            # grok the proxy's URL type
+            # e.g. if we have a constructor arg proxies like so:
+            # {'http': 'ftp://proxy.example.com'}, we may end up turning
+            # a request for http://acme.example.com/a into one for
+            # ftp://proxy.example.com/a
+            return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+    def __init__(self):
+        self.passwd = {}
+
+    def add_password(self, realm, uri, user, passwd):
+        # uri could be a single URI or a sequence
+        if isinstance(uri, str):
+            uri = [uri]
+        if realm not in self.passwd:
+            self.passwd[realm] = {}
+        for default_port in True, False:
+            reduced_uri = tuple(
+                self.reduce_uri(u, default_port) for u in uri)
+            self.passwd[realm][reduced_uri] = (user, passwd)
+
+    def find_user_password(self, realm, authuri):
+        domains = self.passwd.get(realm, {})
+        for default_port in True, False:
+            reduced_authuri = self.reduce_uri(authuri, default_port)
+            for uris, authinfo in domains.items():
+                for uri in uris:
+                    if self.is_suburi(uri, reduced_authuri):
+                        return authinfo
+        return None, None
+
+    def reduce_uri(self, uri, default_port=True):
+        """Accept authority or URI and extract only the authority and path."""
+        # note HTTP URLs do not have a userinfo component
+        parts = urlsplit(uri)
+        if parts[1]:
+            # URI
+            scheme = parts[0]
+            authority = parts[1]
+            path = parts[2] or '/'
+        else:
+            # host or host:port
+            scheme = None
+            authority = uri
+            path = '/'
+        host, port = _splitport(authority)
+        if default_port and port is None and scheme is not None:
+            dport = {"http": 80,
+                     "https": 443,
+                     }.get(scheme)
+            if dport is not None:
+                authority = "%s:%d" % (host, dport)
+        return authority, path
+
+    def is_suburi(self, base, test):
+        """Check if test is below base in a URI tree
+
+        Both args must be URIs in reduced form.
+        """
+        if base == test:
+            return True
+        if base[0] != test[0]:
+            return False
+        prefix = base[1]
+        if prefix[-1:] != '/':
+            prefix += '/'
+        return test[1].startswith(prefix)
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+    def find_user_password(self, realm, authuri):
+        user, password = HTTPPasswordMgr.find_user_password(self, realm,
+                                                            authuri)
+        if user is not None:
+            return user, password
+        return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class HTTPPasswordMgrWithPriorAuth(HTTPPasswordMgrWithDefaultRealm):
+
+    def __init__(self, *args, **kwargs):
+        self.authenticated = {}
+        super().__init__(*args, **kwargs)
+
+    def add_password(self, realm, uri, user, passwd, is_authenticated=False):
+        self.update_authenticated(uri, is_authenticated)
+        # Add a default for prior auth requests
+        if realm is not None:
+            super().add_password(None, uri, user, passwd)
+        super().add_password(realm, uri, user, passwd)
+
+    def update_authenticated(self, uri, is_authenticated=False):
+        # uri could be a single URI or a sequence
+        if isinstance(uri, str):
+            uri = [uri]
+
+        for default_port in True, False:
+            for u in uri:
+                reduced_uri = self.reduce_uri(u, default_port)
+                self.authenticated[reduced_uri] = is_authenticated
+
+    def is_authenticated(self, authuri):
+        for default_port in True, False:
+            reduced_authuri = self.reduce_uri(authuri, default_port)
+            for uri in self.authenticated:
+                if self.is_suburi(uri, reduced_authuri):
+                    return self.authenticated[uri]
+
+
+class AbstractBasicAuthHandler:
+
+    # XXX this allows for multiple auth-schemes, but will stupidly pick
+    # the last one with a realm specified.
+
+    # allow for double- and single-quoted realm values
+    # (single quotes are a violation of the RFC, but appear in the wild)
+    rx = re.compile('(?:^|,)'   # start of the string or ','
+                    '[ \t]*'    # optional whitespaces
+                    '([^ \t,]+)' # scheme like "Basic"
+                    '[ \t]+'    # mandatory whitespaces
+                    # realm=xxx
+                    # realm='xxx'
+                    # realm="xxx"
+                    'realm=(["\']?)([^"\']*)\\2',
+                    re.I)
+
+    # XXX could pre-emptively send auth info already accepted (RFC 2617,
+    # end of section 2, and section 1.2 immediately after "credentials"
+    # production).
+
+    def __init__(self, password_mgr=None):
+        if password_mgr is None:
+            password_mgr = HTTPPasswordMgr()
+        self.passwd = password_mgr
+        self.add_password = self.passwd.add_password
+
+    def _parse_realm(self, header):
+        # parse WWW-Authenticate header: accept multiple challenges per header
+        found_challenge = False
+        for mo in AbstractBasicAuthHandler.rx.finditer(header):
+            scheme, quote, realm = mo.groups()
+            if quote not in ['"', "'"]:
+                warnings.warn("Basic Auth Realm was unquoted",
+                              UserWarning, 3)
+
+            yield (scheme, realm)
+
+            found_challenge = True
+
+        if not found_challenge:
+            if header:
+                scheme = header.split()[0]
+            else:
+                scheme = ''
+            yield (scheme, None)
+
+    def http_error_auth_reqed(self, authreq, host, req, headers):
+        # host may be an authority (without userinfo) or a URL with an
+        # authority
+        headers = headers.get_all(authreq)
+        if not headers:
+            # no header found
+            return
+
+        unsupported = None
+        for header in headers:
+            for scheme, realm in self._parse_realm(header):
+                if scheme.lower() != 'basic':
+                    unsupported = scheme
+                    continue
+
+                if realm is not None:
+                    # Use the first matching Basic challenge.
+                    # Ignore following challenges even if they use the Basic
+                    # scheme.
+                    return self.retry_http_basic_auth(host, req, realm)
+
+        if unsupported is not None:
+            raise ValueError("AbstractBasicAuthHandler does not "
+                             "support the following scheme: %r"
+                             % (scheme,))
+
+    def retry_http_basic_auth(self, host, req, realm):
+        user, pw = self.passwd.find_user_password(realm, host)
+        if pw is not None:
+            raw = "%s:%s" % (user, pw)
+            auth = "Basic " + base64.b64encode(raw.encode()).decode("ascii")
+            if req.get_header(self.auth_header, None) == auth:
+                return None
+            req.add_unredirected_header(self.auth_header, auth)
+            return self.parent.open(req, timeout=req.timeout)
+        else:
+            return None
+
+    def http_request(self, req):
+        if (not hasattr(self.passwd, 'is_authenticated') or
+           not self.passwd.is_authenticated(req.full_url)):
+            return req
+
+        if not req.has_header('Authorization'):
+            user, passwd = self.passwd.find_user_password(None, req.full_url)
+            credentials = '{0}:{1}'.format(user, passwd).encode()
+            auth_str = base64.standard_b64encode(credentials).decode()
+            req.add_unredirected_header('Authorization',
+                                        'Basic {}'.format(auth_str.strip()))
+        return req
+
+    def http_response(self, req, response):
+        if hasattr(self.passwd, 'is_authenticated'):
+            if 200 <= response.code < 300:
+                self.passwd.update_authenticated(req.full_url, True)
+            else:
+                self.passwd.update_authenticated(req.full_url, False)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Authorization'
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        url = req.full_url
+        response = self.http_error_auth_reqed('www-authenticate',
+                                          url, req, headers)
+        return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Proxy-authorization'
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        # http_error_auth_reqed requires that there is no userinfo component in
+        # authority.  Assume there isn't one, since urllib.request does not (and
+        # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+        # userinfo.
+        authority = req.host
+        response = self.http_error_auth_reqed('proxy-authenticate',
+                                          authority, req, headers)
+        return response
+
+
+# Return n random bytes.
+_randombytes = os.urandom
+
+
+class AbstractDigestAuthHandler:
+    # Digest authentication is specified in RFC 2617.
+
+    # XXX The client does not inspect the Authentication-Info header
+    # in a successful response.
+
+    # XXX It should be possible to test this implementation against
+    # a mock server that just generates a static set of challenges.
+
+    # XXX qop="auth-int" supports is shaky
+
+    def __init__(self, passwd=None):
+        if passwd is None:
+            passwd = HTTPPasswordMgr()
+        self.passwd = passwd
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+        self.nonce_count = 0
+        self.last_nonce = None
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, auth_header, host, req, headers):
+        authreq = headers.get(auth_header, None)
+        if self.retried > 5:
+            # Don't fail endlessly - if we failed once, we'll probably
+            # fail a second time. Hm. Unless the Password Manager is
+            # prompting for the information. Crap. This isn't great
+            # but it's better than the current 'repeat until recursion
+            # depth exceeded' approach 
+            raise HTTPError(req.full_url, 401, "digest auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+        if authreq:
+            scheme = authreq.split()[0]
+            if scheme.lower() == 'digest':
+                return self.retry_http_digest_auth(req, authreq)
+            elif scheme.lower() != 'basic':
+                raise ValueError("AbstractDigestAuthHandler does not support"
+                                 " the following scheme: '%s'" % scheme)
+
+    def retry_http_digest_auth(self, req, auth):
+        token, challenge = auth.split(' ', 1)
+        chal = parse_keqv_list(filter(None, parse_http_list(challenge)))
+        auth = self.get_authorization(req, chal)
+        if auth:
+            auth_val = 'Digest %s' % auth
+            if req.headers.get(self.auth_header, None) == auth_val:
+                return None
+            req.add_unredirected_header(self.auth_header, auth_val)
+            resp = self.parent.open(req, timeout=req.timeout)
+            return resp
+
+    def get_cnonce(self, nonce):
+        # The cnonce-value is an opaque
+        # quoted string value provided by the client and used by both client
+        # and server to avoid chosen plaintext attacks, to provide mutual
+        # authentication, and to provide some message integrity protection.
+        # This isn't a fabulous effort, but it's probably Good Enough.
+        s = "%s:%s:%s:" % (self.nonce_count, nonce, time.ctime())
+        b = s.encode("ascii") + _randombytes(8)
+        dig = hashlib.sha1(b).hexdigest()
+        return dig[:16]
+
+    def get_authorization(self, req, chal):
+        try:
+            realm = chal['realm']
+            nonce = chal['nonce']
+            qop = chal.get('qop')
+            algorithm = chal.get('algorithm', 'MD5')
+            # mod_digest doesn't send an opaque, even though it isn't
+            # supposed to be optional
+            opaque = chal.get('opaque', None)
+        except KeyError:
+            return None
+
+        H, KD = self.get_algorithm_impls(algorithm)
+        if H is None:
+            return None
+
+        user, pw = self.passwd.find_user_password(realm, req.full_url)
+        if user is None:
+            return None
+
+        # XXX not implemented yet
+        if req.data is not None:
+            entdig = self.get_entity_digest(req.data, chal)
+        else:
+            entdig = None
+
+        A1 = "%s:%s:%s" % (user, realm, pw)
+        A2 = "%s:%s" % (req.get_method(),
+                        # XXX selector: what about proxies and full urls
+                        req.selector)
+        # NOTE: As per  RFC 2617, when server sends "auth,auth-int", the client could use either `auth`
+        #     or `auth-int` to the response back. we use `auth` to send the response back.
+        if qop is None:
+            respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+        elif 'auth' in qop.split(','):
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+                self.last_nonce = nonce
+            ncvalue = '%08x' % self.nonce_count
+            cnonce = self.get_cnonce(nonce)
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, 'auth', H(A2))
+            respdig = KD(H(A1), noncebit)
+        else:
+            # XXX handle auth-int.
+            raise URLError("qop '%s' is not supported." % qop)
+
+        # XXX should the partial digests be encoded too?
+
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+               'response="%s"' % (user, realm, nonce, req.selector,
+                                  respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+        base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+        return base
+
+    def get_algorithm_impls(self, algorithm):
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            H = lambda x: hashlib.md5(x.encode("ascii")).hexdigest()
+        elif algorithm == 'SHA':
+            H = lambda x: hashlib.sha1(x.encode("ascii")).hexdigest()
+        # XXX MD5-sess
+        else:
+            raise ValueError("Unsupported digest authentication "
+                             "algorithm %r" % algorithm)
+        KD = lambda s, d: H("%s:%s" % (s, d))
+        return H, KD
+
+    def get_entity_digest(self, data, chal):
+        # XXX not implemented yet
+        return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+    """An authentication protocol defined by RFC 2069
+
+    Digest authentication improves on basic authentication because it
+    does not transmit passwords in the clear.
+    """
+
+    auth_header = 'Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        host = urlparse(req.full_url)[1]
+        retry = self.http_error_auth_reqed('www-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+    auth_header = 'Proxy-Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        host = req.host
+        retry = self.http_error_auth_reqed('proxy-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+    def __init__(self, debuglevel=0):
+        self._debuglevel = debuglevel
+
+    def set_http_debuglevel(self, level):
+        self._debuglevel = level
+
+    def _get_content_length(self, request):
+        return http.client.HTTPConnection._get_content_length(
+            request.data,
+            request.get_method())
+
+    def do_request_(self, request):
+        host = request.host
+        if not host:
+            raise URLError('no host given')
+
+        if request.data is not None:  # POST
+            data = request.data
+            if isinstance(data, str):
+                msg = "POST data should be bytes, an iterable of bytes, " \
+                      "or a file object. It cannot be of type str."
+                raise TypeError(msg)
+            if not request.has_header('Content-type'):
+                request.add_unredirected_header(
+                    'Content-type',
+                    'application/x-www-form-urlencoded')
+            if (not request.has_header('Content-length')
+                    and not request.has_header('Transfer-encoding')):
+                content_length = self._get_content_length(request)
+                if content_length is not None:
+                    request.add_unredirected_header(
+                            'Content-length', str(content_length))
+                else:
+                    request.add_unredirected_header(
+                            'Transfer-encoding', 'chunked')
+
+        sel_host = host
+        if request.has_proxy():
+            scheme, sel = _splittype(request.selector)
+            sel_host, sel_path = _splithost(sel)
+        if not request.has_header('Host'):
+            request.add_unredirected_header('Host', sel_host)
+        for name, value in self.parent.addheaders:
+            name = name.capitalize()
+            if not request.has_header(name):
+                request.add_unredirected_header(name, value)
+
+        return request
+
+    def do_open(self, http_class, req, **http_conn_args):
+        """Return an HTTPResponse object for the request, using http_class.
+
+        http_class must implement the HTTPConnection API from http.client.
+        """
+        host = req.host
+        if not host:
+            raise URLError('no host given')
+
+        # will parse host:port
+        h = http_class(host, timeout=req.timeout, **http_conn_args)
+        h.set_debuglevel(self._debuglevel)
+
+        headers = dict(req.unredirected_hdrs)
+        headers.update({k: v for k, v in req.headers.items()
+                        if k not in headers})
+
+        # TODO(jhylton): Should this be redesigned to handle
+        # persistent connections?
+
+        # We want to make an HTTP/1.1 request, but the addinfourl
+        # class isn't prepared to deal with a persistent connection.
+        # It will try to read all remaining data from the socket,
+        # which will block while the server waits for the next request.
+        # So make sure the connection gets closed after the (only)
+        # request.
+        headers["Connection"] = "close"
+        headers = {name.title(): val for name, val in headers.items()}
+
+        if req._tunnel_host:
+            tunnel_headers = {}
+            proxy_auth_hdr = "Proxy-Authorization"
+            if proxy_auth_hdr in headers:
+                tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                # Proxy-Authorization should not be sent to origin
+                # server.
+                del headers[proxy_auth_hdr]
+            h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+        try:
+            try:
+                h.request(req.get_method(), req.selector, req.data, headers,
+                          encode_chunked=req.has_header('Transfer-encoding'))
+            except OSError as err: # timeout error
+                raise URLError(err)
+            r = h.getresponse()
+        except:
+            h.close()
+            raise
+
+        # If the server does not send us a 'Connection: close' header,
+        # HTTPConnection assumes the socket should be left open. Manually
+        # mark the socket to be closed when this response object goes away.
+        if h.sock:
+            h.sock.close()
+            h.sock = None
+
+        r.url = req.get_full_url()
+        # This line replaces the .msg attribute of the HTTPResponse
+        # with .headers, because urllib clients expect the response to
+        # have the reason in .msg.  It would be good to mark this
+        # attribute is deprecated and get then to use info() or
+        # .headers.
+        r.msg = r.reason
+        return r
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+    def http_open(self, req):
+        return self.do_open(http.client.HTTPConnection, req)
+
+    http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(http.client, 'HTTPSConnection'):
+
+    class HTTPSHandler(AbstractHTTPHandler):
+
+        def __init__(self, debuglevel=0, context=None, check_hostname=None):
+            AbstractHTTPHandler.__init__(self, debuglevel)
+            self._context = context
+            self._check_hostname = check_hostname
+
+        def https_open(self, req):
+            return self.do_open(http.client.HTTPSConnection, req,
+                context=self._context, check_hostname=self._check_hostname)
+
+        https_request = AbstractHTTPHandler.do_request_
+
+    __all__.append('HTTPSHandler')
+
+class HTTPCookieProcessor(BaseHandler):
+    def __init__(self, cookiejar=None):
+        import http.cookiejar
+        if cookiejar is None:
+            cookiejar = http.cookiejar.CookieJar()
+        self.cookiejar = cookiejar
+
+    def http_request(self, request):
+        self.cookiejar.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        self.cookiejar.extract_cookies(response, request)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+class UnknownHandler(BaseHandler):
+    def unknown_open(self, req):
+        type = req.type
+        raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+    """Parse list of key=value strings where keys are not duplicated."""
+    parsed = {}
+    for elt in l:
+        k, v = elt.split('=', 1)
+        if v[0] == '"' and v[-1] == '"':
+            v = v[1:-1]
+        parsed[k] = v
+    return parsed
+
+def parse_http_list(s):
+    """Parse lists as described by RFC 2068 Section 2.
+
+    In particular, parse comma-separated lists where the elements of
+    the list may include quoted-strings.  A quoted-string could
+    contain a comma.  A non-quoted string could have quotes in the
+    middle.  Neither commas nor quotes count if they are escaped.
+    Only double-quotes count, not single-quotes.
+    """
+    res = []
+    part = ''
+
+    escape = quote = False
+    for cur in s:
+        if escape:
+            part += cur
+            escape = False
+            continue
+        if quote:
+            if cur == '\\':
+                escape = True
+                continue
+            elif cur == '"':
+                quote = False
+            part += cur
+            continue
+
+        if cur == ',':
+            res.append(part)
+            part = ''
+            continue
+
+        if cur == '"':
+            quote = True
+
+        part += cur
+
+    # append last part
+    if part:
+        res.append(part)
+
+    return [part.strip() for part in res]
+
+class FileHandler(BaseHandler):
+    # Use local file or FTP depending on form of URL
+    def file_open(self, req):
+        url = req.selector
+        if url[:2] == '//' and url[2:3] != '/' and (req.host and
+                req.host != 'localhost'):
+            if not req.host in self.get_names():
+                raise URLError("file:// scheme is supported only on localhost")
+        else:
+            return self.open_local_file(req)
+
+    # names for the localhost
+    names = None
+    def get_names(self):
+        if FileHandler.names is None:
+            try:
+                FileHandler.names = tuple(
+                    socket.gethostbyname_ex('localhost')[2] +
+                    socket.gethostbyname_ex(socket.gethostname())[2])
+            except socket.gaierror:
+                FileHandler.names = (socket.gethostbyname('localhost'),)
+        return FileHandler.names
+
+    # not entirely sure what the rules are here
+    def open_local_file(self, req):
+        import email.utils
+        import mimetypes
+        host = req.host
+        filename = req.selector
+        localfile = url2pathname(filename)
+        try:
+            stats = os.stat(localfile)
+            size = stats.st_size
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            mtype = mimetypes.guess_type(filename)[0]
+            headers = email.message_from_string(
+                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+                (mtype or 'text/plain', size, modified))
+            if host:
+                host, port = _splitport(host)
+            if not host or \
+                (not port and _safe_gethostbyname(host) in self.get_names()):
+                if host:
+                    origurl = 'file://' + host + filename
+                else:
+                    origurl = 'file://' + filename
+                return addinfourl(open(localfile, 'rb'), headers, origurl)
+        except OSError as exp:
+            raise URLError(exp)
+        raise URLError('file not on local host')
+
+def _safe_gethostbyname(host):
+    try:
+        return socket.gethostbyname(host)
+    except socket.gaierror:
+        return None
+
+class FTPHandler(BaseHandler):
+    def ftp_open(self, req):
+        import ftplib
+        import mimetypes
+        host = req.host
+        if not host:
+            raise URLError('ftp error: no host given')
+        host, port = _splitport(host)
+        if port is None:
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+
+        # username/password handling
+        user, host = _splituser(host)
+        if user:
+            user, passwd = _splitpasswd(user)
+        else:
+            passwd = None
+        host = unquote(host)
+        user = user or ''
+        passwd = passwd or ''
+
+        try:
+            host = socket.gethostbyname(host)
+        except OSError as msg:
+            raise URLError(msg)
+        path, attrs = _splitattr(req.selector)
+        dirs = path.split('/')
+        dirs = list(map(unquote, dirs))
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]:
+            dirs = dirs[1:]
+        try:
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+            type = file and 'I' or 'D'
+            for attr in attrs:
+                attr, value = _splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            fp, retrlen = fw.retrfile(file, type)
+            headers = ""
+            mtype = mimetypes.guess_type(req.full_url)[0]
+            if mtype:
+                headers += "Content-type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-length: %d\n" % retrlen
+            headers = email.message_from_string(headers)
+            return addinfourl(fp, headers, req.full_url)
+        except ftplib.all_errors as exp:
+            raise URLError(exp) from exp
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        return ftpwrapper(user, passwd, host, port, dirs, timeout,
+                          persistent=False)
+
+class CacheFTPHandler(FTPHandler):
+    # XXX would be nice to have pluggable cache strategies
+    # XXX this stuff is definitely not thread safe
+    def __init__(self):
+        self.cache = {}
+        self.timeout = {}
+        self.soonest = 0
+        self.delay = 60
+        self.max_conns = 16
+
+    def setTimeout(self, t):
+        self.delay = t
+
+    def setMaxConns(self, m):
+        self.max_conns = m
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        key = user, host, port, '/'.join(dirs), timeout
+        if key in self.cache:
+            self.timeout[key] = time.time() + self.delay
+        else:
+            self.cache[key] = ftpwrapper(user, passwd, host, port,
+                                         dirs, timeout)
+            self.timeout[key] = time.time() + self.delay
+        self.check_cache()
+        return self.cache[key]
+
+    def check_cache(self):
+        # first check for old ones
+        t = time.time()
+        if self.soonest <= t:
+            for k, v in list(self.timeout.items()):
+                if v < t:
+                    self.cache[k].close()
+                    del self.cache[k]
+                    del self.timeout[k]
+        self.soonest = min(list(self.timeout.values()))
+
+        # then check the size
+        if len(self.cache) == self.max_conns:
+            for k, v in list(self.timeout.items()):
+                if v == self.soonest:
+                    del self.cache[k]
+                    del self.timeout[k]
+                    break
+            self.soonest = min(list(self.timeout.values()))
+
+    def clear_cache(self):
+        for conn in self.cache.values():
+            conn.close()
+        self.cache.clear()
+        self.timeout.clear()
+
+class DataHandler(BaseHandler):
+    def data_open(self, req):
+        # data URLs as specified in RFC 2397.
+        #
+        # ignores POSTed data
+        #
+        # syntax:
+        # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
+        # mediatype := [ type "/" subtype ] *( ";" parameter )
+        # data      := *urlchar
+        # parameter := attribute "=" value
+        url = req.full_url
+
+        scheme, data = url.split(":",1)
+        mediatype, data = data.split(",",1)
+
+        # even base64 encoded data URLs might be quoted so unquote in any case:
+        data = unquote_to_bytes(data)
+        if mediatype.endswith(";base64"):
+            data = base64.decodebytes(data)
+            mediatype = mediatype[:-7]
+
+        if not mediatype:
+            mediatype = "text/plain;charset=US-ASCII"
+
+        headers = email.message_from_string("Content-type: %s\nContent-length: %d\n" %
+            (mediatype, len(data)))
+
+        return addinfourl(io.BytesIO(data), headers, url)
+
+
+# Code move from the old urllib module
+
+MAXFTPCACHE = 10        # Trim the ftp cache beyond this size
+
+# Helper for non-unix systems
+if os.name == 'nt':
+    from nturl2path import url2pathname, pathname2url
+else:
+    def url2pathname(pathname):
+        """OS-specific conversion from a relative URL of the 'file' scheme
+        to a file system path; not recommended for general use."""
+        return unquote(pathname)
+
+    def pathname2url(pathname):
+        """OS-specific conversion from a file system path to a relative URL
+        of the 'file' scheme; not recommended for general use."""
+        return quote(pathname)
+
+
+ftpcache = {}
+
+
+class URLopener:
+    """Class to open URLs.
+    This is a class rather than just a subroutine because we may need
+    more than one set of global protocol-specific options.
+    Note -- this is a base class for those who don't want the
+    automatic handling of errors type 302 (relocated) and 401
+    (authorization needed)."""
+
+    __tempfiles = None
+
+    version = "Python-urllib/%s" % __version__
+
+    # Constructor
+    def __init__(self, proxies=None, **x509):
+        msg = "%(class)s style of invoking requests is deprecated. " \
+              "Use newer urlopen functions/methods" % {'class': self.__class__.__name__}
+        warnings.warn(msg, DeprecationWarning, stacklevel=3)
+        if proxies is None:
+            proxies = getproxies()
+        assert hasattr(proxies, 'keys'), "proxies must be a mapping"
+        self.proxies = proxies
+        self.key_file = x509.get('key_file')
+        self.cert_file = x509.get('cert_file')
+        self.addheaders = [('User-Agent', self.version), ('Accept', '*/*')]
+        self.__tempfiles = []
+        self.__unlink = os.unlink # See cleanup()
+        self.tempcache = None
+        # Undocumented feature: if you assign {} to tempcache,
+        # it is used to cache files retrieved with
+        # self.retrieve().  This is not enabled by default
+        # since it does not work for changing documents (and I
+        # haven't got the logic to check expiration headers
+        # yet).
+        self.ftpcache = ftpcache
+        # Undocumented feature: you can use a different
+        # ftp cache by assigning to the .ftpcache member;
+        # in case you want logically independent URL openers
+        # XXX This is not threadsafe.  Bah.
+
+    def __del__(self):
+        self.close()
+
+    def close(self):
+        self.cleanup()
+
+    def cleanup(self):
+        # This code sometimes runs when the rest of this module
+        # has already been deleted, so it can't use any globals
+        # or import anything.
+        if self.__tempfiles:
+            for file in self.__tempfiles:
+                try:
+                    self.__unlink(file)
+                except OSError:
+                    pass
+            del self.__tempfiles[:]
+        if self.tempcache:
+            self.tempcache.clear()
+
+    def addheader(self, *args):
+        """Add a header to be used by the HTTP interface only
+        e.g. u.addheader('Accept', 'sound/basic')"""
+        self.addheaders.append(args)
+
+    # External interface
+    def open(self, fullurl, data=None):
+        """Use URLopener().open(file) instead of open(file, 'r')."""
+        fullurl = unwrap(_to_bytes(fullurl))
+        fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]|")
+        if self.tempcache and fullurl in self.tempcache:
+            filename, headers = self.tempcache[fullurl]
+            fp = open(filename, 'rb')
+            return addinfourl(fp, headers, fullurl)
+        urltype, url = _splittype(fullurl)
+        if not urltype:
+            urltype = 'file'
+        if urltype in self.proxies:
+            proxy = self.proxies[urltype]
+            urltype, proxyhost = _splittype(proxy)
+            host, selector = _splithost(proxyhost)
+            url = (host, fullurl) # Signal special case to open_*()
+        else:
+            proxy = None
+        name = 'open_' + urltype
+        self.type = urltype
+        name = name.replace('-', '_')
+        if not hasattr(self, name) or name == 'open_local_file':
+            if proxy:
+                return self.open_unknown_proxy(proxy, fullurl, data)
+            else:
+                return self.open_unknown(fullurl, data)
+        try:
+            if data is None:
+                return getattr(self, name)(url)
+            else:
+                return getattr(self, name)(url, data)
+        except (HTTPError, URLError):
+            raise
+        except OSError as msg:
+            raise OSError('socket error', msg).with_traceback(sys.exc_info()[2])
+
+    def open_unknown(self, fullurl, data=None):
+        """Overridable interface to open unknown URL type."""
+        type, url = _splittype(fullurl)
+        raise OSError('url error', 'unknown url type', type)
+
+    def open_unknown_proxy(self, proxy, fullurl, data=None):
+        """Overridable interface to open unknown URL type."""
+        type, url = _splittype(fullurl)
+        raise OSError('url error', 'invalid proxy for %s' % type, proxy)
+
+    # External interface
+    def retrieve(self, url, filename=None, reporthook=None, data=None):
+        """retrieve(url) returns (filename, headers) for a local object
+        or (tempfilename, headers) for a remote object."""
+        url = unwrap(_to_bytes(url))
+        if self.tempcache and url in self.tempcache:
+            return self.tempcache[url]
+        type, url1 = _splittype(url)
+        if filename is None and (not type or type == 'file'):
+            try:
+                fp = self.open_local_file(url1)
+                hdrs = fp.info()
+                fp.close()
+                return url2pathname(_splithost(url1)[1]), hdrs
+            except OSError:
+                pass
+        fp = self.open(url, data)
+        try:
+            headers = fp.info()
+            if filename:
+                tfp = open(filename, 'wb')
+            else:
+                garbage, path = _splittype(url)
+                garbage, path = _splithost(path or "")
+                path, garbage = _splitquery(path or "")
+                path, garbage = _splitattr(path or "")
+                suffix = os.path.splitext(path)[1]
+                (fd, filename) = tempfile.mkstemp(suffix)
+                self.__tempfiles.append(filename)
+                tfp = os.fdopen(fd, 'wb')
+            try:
+                result = filename, headers
+                if self.tempcache is not None:
+                    self.tempcache[url] = result
+                bs = 1024*8
+                size = -1
+                read = 0
+                blocknum = 0
+                if "content-length" in headers:
+                    size = int(headers["Content-Length"])
+                if reporthook:
+                    reporthook(blocknum, bs, size)
+                while 1:
+                    block = fp.read(bs)
+                    if not block:
+                        break
+                    read += len(block)
+                    tfp.write(block)
+                    blocknum += 1
+                    if reporthook:
+                        reporthook(blocknum, bs, size)
+            finally:
+                tfp.close()
+        finally:
+            fp.close()
+
+        # raise exception if actual size does not match content-length header
+        if size >= 0 and read < size:
+            raise ContentTooShortError(
+                "retrieval incomplete: got only %i out of %i bytes"
+                % (read, size), result)
+
+        return result
+
+    # Each method named open_ knows how to open that type of URL
+
+    def _open_generic_http(self, connection_factory, url, data):
+        """Make an HTTP connection using connection_class.
+
+        This is an internal method that should be called from
+        open_http() or open_https().
+
+        Arguments:
+        - connection_factory should take a host name and return an
+          HTTPConnection instance.
+        - url is the url to retrieval or a host, relative-path pair.
+        - data is payload for a POST request or None.
+        """
+
+        user_passwd = None
+        proxy_passwd= None
+        if isinstance(url, str):
+            host, selector = _splithost(url)
+            if host:
+                user_passwd, host = _splituser(host)
+                host = unquote(host)
+            realhost = host
+        else:
+            host, selector = url
+            # check whether the proxy contains authorization information
+            proxy_passwd, host = _splituser(host)
+            # now we proceed with the url we want to obtain
+            urltype, rest = _splittype(selector)
+            url = rest
+            user_passwd = None
+            if urltype.lower() != 'http':
+                realhost = None
+            else:
+                realhost, rest = _splithost(rest)
+                if realhost:
+                    user_passwd, realhost = _splituser(realhost)
+                if user_passwd:
+                    selector = "%s://%s%s" % (urltype, realhost, rest)
+                if proxy_bypass(realhost):
+                    host = realhost
+
+        if not host: raise OSError('http error', 'no host given')
+
+        if proxy_passwd:
+            proxy_passwd = unquote(proxy_passwd)
+            proxy_auth = base64.b64encode(proxy_passwd.encode()).decode('ascii')
+        else:
+            proxy_auth = None
+
+        if user_passwd:
+            user_passwd = unquote(user_passwd)
+            auth = base64.b64encode(user_passwd.encode()).decode('ascii')
+        else:
+            auth = None
+        http_conn = connection_factory(host)
+        headers = {}
+        if proxy_auth:
+            headers["Proxy-Authorization"] = "Basic %s" % proxy_auth
+        if auth:
+            headers["Authorization"] =  "Basic %s" % auth
+        if realhost:
+            headers["Host"] = realhost
+
+        # Add Connection:close as we don't support persistent connections yet.
+        # This helps in closing the socket and avoiding ResourceWarning
+
+        headers["Connection"] = "close"
+
+        for header, value in self.addheaders:
+            headers[header] = value
+
+        if data is not None:
+            headers["Content-Type"] = "application/x-www-form-urlencoded"
+            http_conn.request("POST", selector, data, headers)
+        else:
+            http_conn.request("GET", selector, headers=headers)
+
+        try:
+            response = http_conn.getresponse()
+        except http.client.BadStatusLine:
+            # something went wrong with the HTTP status line
+            raise URLError("http protocol error: bad status line")
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if 200 <= response.status < 300:
+            return addinfourl(response, response.msg, "http:" + url,
+                              response.status)
+        else:
+            return self.http_error(
+                url, response.fp,
+                response.status, response.reason, response.msg, data)
+
+    def open_http(self, url, data=None):
+        """Use HTTP protocol."""
+        return self._open_generic_http(http.client.HTTPConnection, url, data)
+
+    def http_error(self, url, fp, errcode, errmsg, headers, data=None):
+        """Handle http errors.
+
+        Derived class can override this, or provide specific handlers
+        named http_error_DDD where DDD is the 3-digit error code."""
+        # First check if there's a specific handler for this error
+        name = 'http_error_%d' % errcode
+        if hasattr(self, name):
+            method = getattr(self, name)
+            if data is None:
+                result = method(url, fp, errcode, errmsg, headers)
+            else:
+                result = method(url, fp, errcode, errmsg, headers, data)
+            if result: return result
+        return self.http_error_default(url, fp, errcode, errmsg, headers)
+
+    def http_error_default(self, url, fp, errcode, errmsg, headers):
+        """Default error handler: close the connection and raise OSError."""
+        fp.close()
+        raise HTTPError(url, errcode, errmsg, headers, None)
+
+    if _have_ssl:
+        def _https_connection(self, host):
+            return http.client.HTTPSConnection(host,
+                                           key_file=self.key_file,
+                                           cert_file=self.cert_file)
+
+        def open_https(self, url, data=None):
+            """Use HTTPS protocol."""
+            return self._open_generic_http(self._https_connection, url, data)
+
+    def open_file(self, url):
+        """Use local file or FTP depending on form of URL."""
+        if not isinstance(url, str):
+            raise URLError('file error: proxy support for file protocol currently not implemented')
+        if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/':
+            raise ValueError("file:// scheme is supported only on localhost")
+        else:
+            return self.open_local_file(url)
+
+    def open_local_file(self, url):
+        """Use local file."""
+        import email.utils
+        import mimetypes
+        host, file = _splithost(url)
+        localname = url2pathname(file)
+        try:
+            stats = os.stat(localname)
+        except OSError as e:
+            raise URLError(e.strerror, e.filename)
+        size = stats.st_size
+        modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+        mtype = mimetypes.guess_type(url)[0]
+        headers = email.message_from_string(
+            'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
+            (mtype or 'text/plain', size, modified))
+        if not host:
+            urlfile = file
+            if file[:1] == '/':
+                urlfile = 'file://' + file
+            return addinfourl(open(localname, 'rb'), headers, urlfile)
+        host, port = _splitport(host)
+        if (not port
+           and socket.gethostbyname(host) in ((localhost(),) + thishost())):
+            urlfile = file
+            if file[:1] == '/':
+                urlfile = 'file://' + file
+            elif file[:2] == './':
+                raise ValueError("local file url may start with / or file:. Unknown url of type: %s" % url)
+            return addinfourl(open(localname, 'rb'), headers, urlfile)
+        raise URLError('local file error: not on local host')
+
+    def open_ftp(self, url):
+        """Use FTP protocol."""
+        if not isinstance(url, str):
+            raise URLError('ftp error: proxy support for ftp protocol currently not implemented')
+        import mimetypes
+        host, path = _splithost(url)
+        if not host: raise URLError('ftp error: no host given')
+        host, port = _splitport(host)
+        user, host = _splituser(host)
+        if user: user, passwd = _splitpasswd(user)
+        else: passwd = None
+        host = unquote(host)
+        user = unquote(user or '')
+        passwd = unquote(passwd or '')
+        host = socket.gethostbyname(host)
+        if not port:
+            import ftplib
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+        path, attrs = _splitattr(path)
+        path = unquote(path)
+        dirs = path.split('/')
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]: dirs = dirs[1:]
+        if dirs and not dirs[0]: dirs[0] = '/'
+        key = user, host, port, '/'.join(dirs)
+        # XXX thread unsafe!
+        if len(self.ftpcache) > MAXFTPCACHE:
+            # Prune the cache, rather arbitrarily
+            for k in list(self.ftpcache):
+                if k != key:
+                    v = self.ftpcache[k]
+                    del self.ftpcache[k]
+                    v.close()
+        try:
+            if key not in self.ftpcache:
+                self.ftpcache[key] = \
+                    ftpwrapper(user, passwd, host, port, dirs)
+            if not file: type = 'D'
+            else: type = 'I'
+            for attr in attrs:
+                attr, value = _splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            (fp, retrlen) = self.ftpcache[key].retrfile(file, type)
+            mtype = mimetypes.guess_type("ftp:" + url)[0]
+            headers = ""
+            if mtype:
+                headers += "Content-Type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-Length: %d\n" % retrlen
+            headers = email.message_from_string(headers)
+            return addinfourl(fp, headers, "ftp:" + url)
+        except ftperrors() as exp:
+            raise URLError('ftp error %r' % exp).with_traceback(sys.exc_info()[2])
+
+    def open_data(self, url, data=None):
+        """Use "data" URL."""
+        if not isinstance(url, str):
+            raise URLError('data error: proxy support for data protocol currently not implemented')
+        # ignore POSTed data
+        #
+        # syntax of data URLs:
+        # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
+        # mediatype := [ type "/" subtype ] *( ";" parameter )
+        # data      := *urlchar
+        # parameter := attribute "=" value
+        try:
+            [type, data] = url.split(',', 1)
+        except ValueError:
+            raise OSError('data error', 'bad data URL')
+        if not type:
+            type = 'text/plain;charset=US-ASCII'
+        semi = type.rfind(';')
+        if semi >= 0 and '=' not in type[semi:]:
+            encoding = type[semi+1:]
+            type = type[:semi]
+        else:
+            encoding = ''
+        msg = []
+        msg.append('Date: %s'%time.strftime('%a, %d %b %Y %H:%M:%S GMT',
+                                            time.gmtime(time.time())))
+        msg.append('Content-type: %s' % type)
+        if encoding == 'base64':
+            # XXX is this encoding/decoding ok?
+            data = base64.decodebytes(data.encode('ascii')).decode('latin-1')
+        else:
+            data = unquote(data)
+        msg.append('Content-Length: %d' % len(data))
+        msg.append('')
+        msg.append(data)
+        msg = '\n'.join(msg)
+        headers = email.message_from_string(msg)
+        f = io.StringIO(msg)
+        #f.fileno = None     # needed for addinfourl
+        return addinfourl(f, headers, url)
+
+
+class FancyURLopener(URLopener):
+    """Derived class with handlers for errors we can handle (perhaps)."""
+
+    def __init__(self, *args, **kwargs):
+        URLopener.__init__(self, *args, **kwargs)
+        self.auth_cache = {}
+        self.tries = 0
+        self.maxtries = 10
+
+    def http_error_default(self, url, fp, errcode, errmsg, headers):
+        """Default error handling -- don't raise an exception."""
+        return addinfourl(fp, headers, "http:" + url, errcode)
+
+    def http_error_302(self, url, fp, errcode, errmsg, headers, data=None):
+        """Error 302 -- relocated (temporarily)."""
+        self.tries += 1
+        try:
+            if self.maxtries and self.tries >= self.maxtries:
+                if hasattr(self, "http_error_500"):
+                    meth = self.http_error_500
+                else:
+                    meth = self.http_error_default
+                return meth(url, fp, 500,
+                            "Internal Server Error: Redirect Recursion",
+                            headers)
+            result = self.redirect_internal(url, fp, errcode, errmsg,
+                                            headers, data)
+            return result
+        finally:
+            self.tries = 0
+
+    def redirect_internal(self, url, fp, errcode, errmsg, headers, data):
+        if 'location' in headers:
+            newurl = headers['location']
+        elif 'uri' in headers:
+            newurl = headers['uri']
+        else:
+            return
+        fp.close()
+
+        # In case the server sent a relative URL, join with original:
+        newurl = urljoin(self.type + ":" + url, newurl)
+
+        urlparts = urlparse(newurl)
+
+        # For security reasons, we don't allow redirection to anything other
+        # than http, https and ftp.
+
+        # We are using newer HTTPError with older redirect_internal method
+        # This older method will get deprecated in 3.3
+
+        if urlparts.scheme not in ('http', 'https', 'ftp', ''):
+            raise HTTPError(newurl, errcode,
+                            errmsg +
+                            " Redirection to url '%s' is not allowed." % newurl,
+                            headers, fp)
+
+        return self.open(newurl)
+
+    def http_error_301(self, url, fp, errcode, errmsg, headers, data=None):
+        """Error 301 -- also relocated (permanently)."""
+        return self.http_error_302(url, fp, errcode, errmsg, headers, data)
+
+    def http_error_303(self, url, fp, errcode, errmsg, headers, data=None):
+        """Error 303 -- also relocated (essentially identical to 302)."""
+        return self.http_error_302(url, fp, errcode, errmsg, headers, data)
+
+    def http_error_307(self, url, fp, errcode, errmsg, headers, data=None):
+        """Error 307 -- relocated, but turn POST into error."""
+        if data is None:
+            return self.http_error_302(url, fp, errcode, errmsg, headers, data)
+        else:
+            return self.http_error_default(url, fp, errcode, errmsg, headers)
+
+    def http_error_401(self, url, fp, errcode, errmsg, headers, data=None,
+            retry=False):
+        """Error 401 -- authentication required.
+        This function supports Basic authentication only."""
+        if 'www-authenticate' not in headers:
+            URLopener.http_error_default(self, url, fp,
+                                         errcode, errmsg, headers)
+        stuff = headers['www-authenticate']
+        match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
+        if not match:
+            URLopener.http_error_default(self, url, fp,
+                                         errcode, errmsg, headers)
+        scheme, realm = match.groups()
+        if scheme.lower() != 'basic':
+            URLopener.http_error_default(self, url, fp,
+                                         errcode, errmsg, headers)
+        if not retry:
+            URLopener.http_error_default(self, url, fp, errcode, errmsg,
+                    headers)
+        name = 'retry_' + self.type + '_basic_auth'
+        if data is None:
+            return getattr(self,name)(url, realm)
+        else:
+            return getattr(self,name)(url, realm, data)
+
+    def http_error_407(self, url, fp, errcode, errmsg, headers, data=None,
+            retry=False):
+        """Error 407 -- proxy authentication required.
+        This function supports Basic authentication only."""
+        if 'proxy-authenticate' not in headers:
+            URLopener.http_error_default(self, url, fp,
+                                         errcode, errmsg, headers)
+        stuff = headers['proxy-authenticate']
+        match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
+        if not match:
+            URLopener.http_error_default(self, url, fp,
+                                         errcode, errmsg, headers)
+        scheme, realm = match.groups()
+        if scheme.lower() != 'basic':
+            URLopener.http_error_default(self, url, fp,
+                                         errcode, errmsg, headers)
+        if not retry:
+            URLopener.http_error_default(self, url, fp, errcode, errmsg,
+                    headers)
+        name = 'retry_proxy_' + self.type + '_basic_auth'
+        if data is None:
+            return getattr(self,name)(url, realm)
+        else:
+            return getattr(self,name)(url, realm, data)
+
+    def retry_proxy_http_basic_auth(self, url, realm, data=None):
+        host, selector = _splithost(url)
+        newurl = 'http://' + host + selector
+        proxy = self.proxies['http']
+        urltype, proxyhost = _splittype(proxy)
+        proxyhost, proxyselector = _splithost(proxyhost)
+        i = proxyhost.find('@') + 1
+        proxyhost = proxyhost[i:]
+        user, passwd = self.get_user_passwd(proxyhost, realm, i)
+        if not (user or passwd): return None
+        proxyhost = "%s:%s@%s" % (quote(user, safe=''),
+                                  quote(passwd, safe=''), proxyhost)
+        self.proxies['http'] = 'http://' + proxyhost + proxyselector
+        if data is None:
+            return self.open(newurl)
+        else:
+            return self.open(newurl, data)
+
+    def retry_proxy_https_basic_auth(self, url, realm, data=None):
+        host, selector = _splithost(url)
+        newurl = 'https://' + host + selector
+        proxy = self.proxies['https']
+        urltype, proxyhost = _splittype(proxy)
+        proxyhost, proxyselector = _splithost(proxyhost)
+        i = proxyhost.find('@') + 1
+        proxyhost = proxyhost[i:]
+        user, passwd = self.get_user_passwd(proxyhost, realm, i)
+        if not (user or passwd): return None
+        proxyhost = "%s:%s@%s" % (quote(user, safe=''),
+                                  quote(passwd, safe=''), proxyhost)
+        self.proxies['https'] = 'https://' + proxyhost + proxyselector
+        if data is None:
+            return self.open(newurl)
+        else:
+            return self.open(newurl, data)
+
+    def retry_http_basic_auth(self, url, realm, data=None):
+        host, selector = _splithost(url)
+        i = host.find('@') + 1
+        host = host[i:]
+        user, passwd = self.get_user_passwd(host, realm, i)
+        if not (user or passwd): return None
+        host = "%s:%s@%s" % (quote(user, safe=''),
+                             quote(passwd, safe=''), host)
+        newurl = 'http://' + host + selector
+        if data is None:
+            return self.open(newurl)
+        else:
+            return self.open(newurl, data)
+
+    def retry_https_basic_auth(self, url, realm, data=None):
+        host, selector = _splithost(url)
+        i = host.find('@') + 1
+        host = host[i:]
+        user, passwd = self.get_user_passwd(host, realm, i)
+        if not (user or passwd): return None
+        host = "%s:%s@%s" % (quote(user, safe=''),
+                             quote(passwd, safe=''), host)
+        newurl = 'https://' + host + selector
+        if data is None:
+            return self.open(newurl)
+        else:
+            return self.open(newurl, data)
+
+    def get_user_passwd(self, host, realm, clear_cache=0):
+        key = realm + '@' + host.lower()
+        if key in self.auth_cache:
+            if clear_cache:
+                del self.auth_cache[key]
+            else:
+                return self.auth_cache[key]
+        user, passwd = self.prompt_user_passwd(host, realm)
+        if user or passwd: self.auth_cache[key] = (user, passwd)
+        return user, passwd
+
+    def prompt_user_passwd(self, host, realm):
+        """Override this in a GUI environment!"""
+        import getpass
+        try:
+            user = input("Enter username for %s at %s: " % (realm, host))
+            passwd = getpass.getpass("Enter password for %s in %s at %s: " %
+                (user, realm, host))
+            return user, passwd
+        except KeyboardInterrupt:
+            print()
+            return None, None
+
+
+# Utility functions
+
+_localhost = None
+def localhost():
+    """Return the IP address of the magic hostname 'localhost'."""
+    global _localhost
+    if _localhost is None:
+        _localhost = socket.gethostbyname('localhost')
+    return _localhost
+
+_thishost = None
+def thishost():
+    """Return the IP addresses of the current host."""
+    global _thishost
+    if _thishost is None:
+        try:
+            _thishost = tuple(socket.gethostbyname_ex(socket.gethostname())[2])
+        except socket.gaierror:
+            _thishost = tuple(socket.gethostbyname_ex('localhost')[2])
+    return _thishost
+
+_ftperrors = None
+def ftperrors():
+    """Return the set of errors raised by the FTP class."""
+    global _ftperrors
+    if _ftperrors is None:
+        import ftplib
+        _ftperrors = ftplib.all_errors
+    return _ftperrors
+
+_noheaders = None
+def noheaders():
+    """Return an empty email Message object."""
+    global _noheaders
+    if _noheaders is None:
+        _noheaders = email.message_from_string("")
+    return _noheaders
+
+
+# Utility classes
+
+class ftpwrapper:
+    """Class used by open_ftp() for cache of open FTP connections."""
+
+    def __init__(self, user, passwd, host, port, dirs, timeout=None,
+                 persistent=True):
+        self.user = user
+        self.passwd = passwd
+        self.host = host
+        self.port = port
+        self.dirs = dirs
+        self.timeout = timeout
+        self.refcount = 0
+        self.keepalive = persistent
+        try:
+            self.init()
+        except:
+            self.close()
+            raise
+
+    def init(self):
+        import ftplib
+        self.busy = 0
+        self.ftp = ftplib.FTP()
+        self.ftp.connect(self.host, self.port, self.timeout)
+        self.ftp.login(self.user, self.passwd)
+        _target = '/'.join(self.dirs)
+        self.ftp.cwd(_target)
+
+    def retrfile(self, file, type):
+        import ftplib
+        self.endtransfer()
+        if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
+        else: cmd = 'TYPE ' + type; isdir = 0
+        try:
+            self.ftp.voidcmd(cmd)
+        except ftplib.all_errors:
+            self.init()
+            self.ftp.voidcmd(cmd)
+        conn = None
+        if file and not isdir:
+            # Try to retrieve as a file
+            try:
+                cmd = 'RETR ' + file
+                conn, retrlen = self.ftp.ntransfercmd(cmd)
+            except ftplib.error_perm as reason:
+                if str(reason)[:3] != '550':
+                    raise URLError('ftp error: %r' % reason).with_traceback(
+                        sys.exc_info()[2])
+        if not conn:
+            # Set transfer mode to ASCII!
+            self.ftp.voidcmd('TYPE A')
+            # Try a directory listing. Verify that directory exists.
+            if file:
+                pwd = self.ftp.pwd()
+                try:
+                    try:
+                        self.ftp.cwd(file)
+                    except ftplib.error_perm as reason:
+                        raise URLError('ftp error: %r' % reason) from reason
+                finally:
+                    self.ftp.cwd(pwd)
+                cmd = 'LIST ' + file
+            else:
+                cmd = 'LIST'
+            conn, retrlen = self.ftp.ntransfercmd(cmd)
+        self.busy = 1
+
+        ftpobj = addclosehook(conn.makefile('rb'), self.file_close)
+        self.refcount += 1
+        conn.close()
+        # Pass back both a suitably decorated object and a retrieval length
+        return (ftpobj, retrlen)
+
+    def endtransfer(self):
+        self.busy = 0
+
+    def close(self):
+        self.keepalive = False
+        if self.refcount <= 0:
+            self.real_close()
+
+    def file_close(self):
+        self.endtransfer()
+        self.refcount -= 1
+        if self.refcount <= 0 and not self.keepalive:
+            self.real_close()
+
+    def real_close(self):
+        self.endtransfer()
+        try:
+            self.ftp.close()
+        except ftperrors():
+            pass
+
+# Proxy handling
+def getproxies_environment():
+    """Return a dictionary of scheme -> proxy server URL mappings.
+
+    Scan the environment for variables named _proxy;
+    this seems to be the standard convention.  If you need a
+    different way, you can pass a proxies dictionary to the
+    [Fancy]URLopener constructor.
+    """
+    # in order to prefer lowercase variables, process environment in
+    # two passes: first matches any, second pass matches lowercase only
+
+    # select only environment variables which end in (after making lowercase) _proxy
+    proxies = {}
+    environment = []
+    for name in os.environ.keys():
+        # fast screen underscore position before more expensive case-folding
+        if len(name) > 5 and name[-6] == "_" and name[-5:].lower() == "proxy":
+            value = os.environ[name]
+            proxy_name = name[:-6].lower()
+            environment.append((name, value, proxy_name))
+            if value:
+                proxies[proxy_name] = value
+    # CVE-2016-1000110 - If we are running as CGI script, forget HTTP_PROXY
+    # (non-all-lowercase) as it may be set from the web server by a "Proxy:"
+    # header from the client
+    # If "proxy" is lowercase, it will still be used thanks to the next block
+    if 'REQUEST_METHOD' in os.environ:
+        proxies.pop('http', None)
+    for name, value, proxy_name in environment:
+        # not case-folded, checking here for lower-case env vars only
+        if name[-6:] == '_proxy':
+            if value:
+                proxies[proxy_name] = value
+            else:
+                proxies.pop(proxy_name, None)
+    return proxies
+
+def proxy_bypass_environment(host, proxies=None):
+    """Test if proxies should not be used for a particular host.
+
+    Checks the proxy dict for the value of no_proxy, which should
+    be a list of comma separated DNS suffixes, or '*' for all hosts.
+
+    """
+    if proxies is None:
+        proxies = getproxies_environment()
+    # don't bypass, if no_proxy isn't specified
+    try:
+        no_proxy = proxies['no']
+    except KeyError:
+        return False
+    # '*' is special case for always bypass
+    if no_proxy == '*':
+        return True
+    host = host.lower()
+    # strip port off host
+    hostonly, port = _splitport(host)
+    # check if the host ends with any of the DNS suffixes
+    for name in no_proxy.split(','):
+        name = name.strip()
+        if name:
+            name = name.lstrip('.')  # ignore leading dots
+            name = name.lower()
+            if hostonly == name or host == name:
+                return True
+            name = '.' + name
+            if hostonly.endswith(name) or host.endswith(name):
+                return True
+    # otherwise, don't bypass
+    return False
+
+
+# This code tests an OSX specific data structure but is testable on all
+# platforms
+def _proxy_bypass_macosx_sysconf(host, proxy_settings):
+    """
+    Return True iff this host shouldn't be accessed using a proxy
+
+    This function uses the MacOSX framework SystemConfiguration
+    to fetch the proxy information.
+
+    proxy_settings come from _scproxy._get_proxy_settings or get mocked ie:
+    { 'exclude_simple': bool,
+      'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.1', '10.0/16']
+    }
+    """
+    from fnmatch import fnmatch
+    from ipaddress import AddressValueError, IPv4Address
+
+    hostonly, port = _splitport(host)
+
+    def ip2num(ipAddr):
+        parts = ipAddr.split('.')
+        parts = list(map(int, parts))
+        if len(parts) != 4:
+            parts = (parts + [0, 0, 0, 0])[:4]
+        return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]
+
+    # Check for simple host names:
+    if '.' not in host:
+        if proxy_settings['exclude_simple']:
+            return True
+
+    hostIP = None
+    try:
+        hostIP = int(IPv4Address(hostonly))
+    except AddressValueError:
+        pass
+
+    for value in proxy_settings.get('exceptions', ()):
+        # Items in the list are strings like these: *.local, 169.254/16
+        if not value: continue
+
+        m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
+        if m is not None and hostIP is not None:
+            base = ip2num(m.group(1))
+            mask = m.group(2)
+            if mask is None:
+                mask = 8 * (m.group(1).count('.') + 1)
+            else:
+                mask = int(mask[1:])
+
+            if mask < 0 or mask > 32:
+                # System libraries ignore invalid prefix lengths
+                continue
+
+            mask = 32 - mask
+
+            if (hostIP >> mask) == (base >> mask):
+                return True
+
+        elif fnmatch(host, value):
+            return True
+
+    return False
+
+
+# Same as _proxy_bypass_macosx_sysconf, testable on all platforms
+def _proxy_bypass_winreg_override(host, override):
+    """Return True if the host should bypass the proxy server.
+
+    The proxy override list is obtained from the Windows
+    Internet settings proxy override registry value.
+
+    An example of a proxy override value is:
+    "www.example.com;*.example.net; 192.168.0.1"
+    """
+    from fnmatch import fnmatch
+
+    host, _ = _splitport(host)
+    proxy_override = override.split(';')
+    for test in proxy_override:
+        test = test.strip()
+        # "" should bypass the proxy server for all intranet addresses
+        if test == '':
+            if '.' not in host:
+                return True
+        elif fnmatch(host, test):
+            return True
+    return False
+
+
+if sys.platform == 'darwin':
+    from _scproxy import _get_proxy_settings, _get_proxies
+
+    def proxy_bypass_macosx_sysconf(host):
+        proxy_settings = _get_proxy_settings()
+        return _proxy_bypass_macosx_sysconf(host, proxy_settings)
+
+    def getproxies_macosx_sysconf():
+        """Return a dictionary of scheme -> proxy server URL mappings.
+
+        This function uses the MacOSX framework SystemConfiguration
+        to fetch the proxy information.
+        """
+        return _get_proxies()
+
+
+
+    def proxy_bypass(host):
+        """Return True, if host should be bypassed.
+
+        Checks proxy settings gathered from the environment, if specified,
+        or from the MacOSX framework SystemConfiguration.
+
+        """
+        proxies = getproxies_environment()
+        if proxies:
+            return proxy_bypass_environment(host, proxies)
+        else:
+            return proxy_bypass_macosx_sysconf(host)
+
+    def getproxies():
+        return getproxies_environment() or getproxies_macosx_sysconf()
+
+
+elif os.name == 'nt':
+    def getproxies_registry():
+        """Return a dictionary of scheme -> proxy server URL mappings.
+
+        Win32 uses the registry to store proxies.
+
+        """
+        proxies = {}
+        try:
+            import winreg
+        except ImportError:
+            # Std module, so should be around - but you never know!
+            return proxies
+        try:
+            internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
+                r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
+            proxyEnable = winreg.QueryValueEx(internetSettings,
+                                               'ProxyEnable')[0]
+            if proxyEnable:
+                # Returned as Unicode but problems if not converted to ASCII
+                proxyServer = str(winreg.QueryValueEx(internetSettings,
+                                                       'ProxyServer')[0])
+                if '=' not in proxyServer and ';' not in proxyServer:
+                    # Use one setting for all protocols.
+                    proxyServer = 'http={0};https={0};ftp={0}'.format(proxyServer)
+                for p in proxyServer.split(';'):
+                    protocol, address = p.split('=', 1)
+                    # See if address has a type:// prefix
+                    if not re.match('(?:[^/:]+)://', address):
+                        # Add type:// prefix to address without specifying type
+                        if protocol in ('http', 'https', 'ftp'):
+                            # The default proxy type of Windows is HTTP
+                            address = 'http://' + address
+                        elif protocol == 'socks':
+                            address = 'socks://' + address
+                    proxies[protocol] = address
+                # Use SOCKS proxy for HTTP(S) protocols
+                if proxies.get('socks'):
+                    # The default SOCKS proxy type of Windows is SOCKS4
+                    address = re.sub(r'^socks://', 'socks4://', proxies['socks'])
+                    proxies['http'] = proxies.get('http') or address
+                    proxies['https'] = proxies.get('https') or address
+            internetSettings.Close()
+        except (OSError, ValueError, TypeError):
+            # Either registry key not found etc, or the value in an
+            # unexpected format.
+            # proxies already set up to be empty so nothing to do
+            pass
+        return proxies
+
+    def getproxies():
+        """Return a dictionary of scheme -> proxy server URL mappings.
+
+        Returns settings gathered from the environment, if specified,
+        or the registry.
+
+        """
+        return getproxies_environment() or getproxies_registry()
+
+    def proxy_bypass_registry(host):
+        try:
+            import winreg
+        except ImportError:
+            # Std modules, so should be around - but you never know!
+            return False
+        try:
+            internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
+                r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
+            proxyEnable = winreg.QueryValueEx(internetSettings,
+                                               'ProxyEnable')[0]
+            proxyOverride = str(winreg.QueryValueEx(internetSettings,
+                                                     'ProxyOverride')[0])
+            # ^^^^ Returned as Unicode but problems if not converted to ASCII
+        except OSError:
+            return False
+        if not proxyEnable or not proxyOverride:
+            return False
+        return _proxy_bypass_winreg_override(host, proxyOverride)
+
+    def proxy_bypass(host):
+        """Return True, if host should be bypassed.
+
+        Checks proxy settings gathered from the environment, if specified,
+        or the registry.
+
+        """
+        proxies = getproxies_environment()
+        if proxies:
+            return proxy_bypass_environment(host, proxies)
+        else:
+            return proxy_bypass_registry(host)
+
+else:
+    # By default use environment variables
+    getproxies = getproxies_environment
+    proxy_bypass = proxy_bypass_environment
diff --git a/lib/python3.1/urllib/response.py b/lib/python3.1/urllib/response.py
new file mode 100644
index 0000000000000000000000000000000000000000..5a2c3cc78c395d57de3fbadf7d75b96035b785b6
--- /dev/null
+++ b/lib/python3.1/urllib/response.py
@@ -0,0 +1,84 @@
+"""Response classes used by urllib.
+
+The base class, addbase, defines a minimal file-like interface,
+including read() and readline().  The typical response object is an
+addinfourl instance, which defines an info() method that returns
+headers and a geturl() method that returns the url.
+"""
+
+import tempfile
+
+__all__ = ['addbase', 'addclosehook', 'addinfo', 'addinfourl']
+
+
+class addbase(tempfile._TemporaryFileWrapper):
+    """Base class for addinfo and addclosehook. Is a good idea for garbage collection."""
+
+    # XXX Add a method to expose the timeout on the underlying socket?
+
+    def __init__(self, fp):
+        super(addbase,  self).__init__(fp, '', delete=False)
+        # Keep reference around as this was part of the original API.
+        self.fp = fp
+
+    def __repr__(self):
+        return '<%s at %r whose fp = %r>' % (self.__class__.__name__,
+                                             id(self), self.file)
+
+    def __enter__(self):
+        if self.fp.closed:
+            raise ValueError("I/O operation on closed file")
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.close()
+
+
+class addclosehook(addbase):
+    """Class to add a close hook to an open file."""
+
+    def __init__(self, fp, closehook, *hookargs):
+        super(addclosehook, self).__init__(fp)
+        self.closehook = closehook
+        self.hookargs = hookargs
+
+    def close(self):
+        try:
+            closehook = self.closehook
+            hookargs = self.hookargs
+            if closehook:
+                self.closehook = None
+                self.hookargs = None
+                closehook(*hookargs)
+        finally:
+            super(addclosehook, self).close()
+
+
+class addinfo(addbase):
+    """class to add an info() method to an open file."""
+
+    def __init__(self, fp, headers):
+        super(addinfo, self).__init__(fp)
+        self.headers = headers
+
+    def info(self):
+        return self.headers
+
+
+class addinfourl(addinfo):
+    """class to add info() and geturl() methods to an open file."""
+
+    def __init__(self, fp, headers, url, code=None):
+        super(addinfourl, self).__init__(fp, headers)
+        self.url = url
+        self.code = code
+
+    @property
+    def status(self):
+        return self.code
+
+    def getcode(self):
+        return self.code
+
+    def geturl(self):
+        return self.url
diff --git a/lib/python3.1/urllib/robotparser.py b/lib/python3.1/urllib/robotparser.py
new file mode 100644
index 0000000000000000000000000000000000000000..c58565e39451461f03dc7d9fb652554c96615f00
--- /dev/null
+++ b/lib/python3.1/urllib/robotparser.py
@@ -0,0 +1,273 @@
+""" robotparser.py
+
+    Copyright (C) 2000  Bastian Kleineidam
+
+    You can choose between two licenses when using this package:
+    1) GNU GPLv2
+    2) PSF license for Python 2.2
+
+    The robots.txt Exclusion Protocol is implemented as specified in
+    http://www.robotstxt.org/norobots-rfc.txt
+"""
+
+import collections
+import urllib.parse
+import urllib.request
+
+__all__ = ["RobotFileParser"]
+
+RequestRate = collections.namedtuple("RequestRate", "requests seconds")
+
+
+class RobotFileParser:
+    """ This class provides a set of methods to read, parse and answer
+    questions about a single robots.txt file.
+
+    """
+
+    def __init__(self, url=''):
+        self.entries = []
+        self.sitemaps = []
+        self.default_entry = None
+        self.disallow_all = False
+        self.allow_all = False
+        self.set_url(url)
+        self.last_checked = 0
+
+    def mtime(self):
+        """Returns the time the robots.txt file was last fetched.
+
+        This is useful for long-running web spiders that need to
+        check for new robots.txt files periodically.
+
+        """
+        return self.last_checked
+
+    def modified(self):
+        """Sets the time the robots.txt file was last fetched to the
+        current time.
+
+        """
+        import time
+        self.last_checked = time.time()
+
+    def set_url(self, url):
+        """Sets the URL referring to a robots.txt file."""
+        self.url = url
+        self.host, self.path = urllib.parse.urlparse(url)[1:3]
+
+    def read(self):
+        """Reads the robots.txt URL and feeds it to the parser."""
+        try:
+            f = urllib.request.urlopen(self.url)
+        except urllib.error.HTTPError as err:
+            if err.code in (401, 403):
+                self.disallow_all = True
+            elif err.code >= 400 and err.code < 500:
+                self.allow_all = True
+        else:
+            raw = f.read()
+            self.parse(raw.decode("utf-8").splitlines())
+
+    def _add_entry(self, entry):
+        if "*" in entry.useragents:
+            # the default entry is considered last
+            if self.default_entry is None:
+                # the first default entry wins
+                self.default_entry = entry
+        else:
+            self.entries.append(entry)
+
+    def parse(self, lines):
+        """Parse the input lines from a robots.txt file.
+
+        We allow that a user-agent: line is not preceded by
+        one or more blank lines.
+        """
+        # states:
+        #   0: start state
+        #   1: saw user-agent line
+        #   2: saw an allow or disallow line
+        state = 0
+        entry = Entry()
+
+        self.modified()
+        for line in lines:
+            if not line:
+                if state == 1:
+                    entry = Entry()
+                    state = 0
+                elif state == 2:
+                    self._add_entry(entry)
+                    entry = Entry()
+                    state = 0
+            # remove optional comment and strip line
+            i = line.find('#')
+            if i >= 0:
+                line = line[:i]
+            line = line.strip()
+            if not line:
+                continue
+            line = line.split(':', 1)
+            if len(line) == 2:
+                line[0] = line[0].strip().lower()
+                line[1] = urllib.parse.unquote(line[1].strip())
+                if line[0] == "user-agent":
+                    if state == 2:
+                        self._add_entry(entry)
+                        entry = Entry()
+                    entry.useragents.append(line[1])
+                    state = 1
+                elif line[0] == "disallow":
+                    if state != 0:
+                        entry.rulelines.append(RuleLine(line[1], False))
+                        state = 2
+                elif line[0] == "allow":
+                    if state != 0:
+                        entry.rulelines.append(RuleLine(line[1], True))
+                        state = 2
+                elif line[0] == "crawl-delay":
+                    if state != 0:
+                        # before trying to convert to int we need to make
+                        # sure that robots.txt has valid syntax otherwise
+                        # it will crash
+                        if line[1].strip().isdigit():
+                            entry.delay = int(line[1])
+                        state = 2
+                elif line[0] == "request-rate":
+                    if state != 0:
+                        numbers = line[1].split('/')
+                        # check if all values are sane
+                        if (len(numbers) == 2 and numbers[0].strip().isdigit()
+                            and numbers[1].strip().isdigit()):
+                            entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1]))
+                        state = 2
+                elif line[0] == "sitemap":
+                    # According to http://www.sitemaps.org/protocol.html
+                    # "This directive is independent of the user-agent line,
+                    #  so it doesn't matter where you place it in your file."
+                    # Therefore we do not change the state of the parser.
+                    self.sitemaps.append(line[1])
+        if state == 2:
+            self._add_entry(entry)
+
+    def can_fetch(self, useragent, url):
+        """using the parsed robots.txt decide if useragent can fetch url"""
+        if self.disallow_all:
+            return False
+        if self.allow_all:
+            return True
+        # Until the robots.txt file has been read or found not
+        # to exist, we must assume that no url is allowable.
+        # This prevents false positives when a user erroneously
+        # calls can_fetch() before calling read().
+        if not self.last_checked:
+            return False
+        # search for given user agent matches
+        # the first match counts
+        parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url))
+        url = urllib.parse.urlunparse(('','',parsed_url.path,
+            parsed_url.params,parsed_url.query, parsed_url.fragment))
+        url = urllib.parse.quote(url)
+        if not url:
+            url = "/"
+        for entry in self.entries:
+            if entry.applies_to(useragent):
+                return entry.allowance(url)
+        # try the default entry last
+        if self.default_entry:
+            return self.default_entry.allowance(url)
+        # agent not found ==> access granted
+        return True
+
+    def crawl_delay(self, useragent):
+        if not self.mtime():
+            return None
+        for entry in self.entries:
+            if entry.applies_to(useragent):
+                return entry.delay
+        if self.default_entry:
+            return self.default_entry.delay
+        return None
+
+    def request_rate(self, useragent):
+        if not self.mtime():
+            return None
+        for entry in self.entries:
+            if entry.applies_to(useragent):
+                return entry.req_rate
+        if self.default_entry:
+            return self.default_entry.req_rate
+        return None
+
+    def site_maps(self):
+        if not self.sitemaps:
+            return None
+        return self.sitemaps
+
+    def __str__(self):
+        entries = self.entries
+        if self.default_entry is not None:
+            entries = entries + [self.default_entry]
+        return '\n\n'.join(map(str, entries))
+
+
+class RuleLine:
+    """A rule line is a single "Allow:" (allowance==True) or "Disallow:"
+       (allowance==False) followed by a path."""
+    def __init__(self, path, allowance):
+        if path == '' and not allowance:
+            # an empty value means allow all
+            allowance = True
+        path = urllib.parse.urlunparse(urllib.parse.urlparse(path))
+        self.path = urllib.parse.quote(path)
+        self.allowance = allowance
+
+    def applies_to(self, filename):
+        return self.path == "*" or filename.startswith(self.path)
+
+    def __str__(self):
+        return ("Allow" if self.allowance else "Disallow") + ": " + self.path
+
+
+class Entry:
+    """An entry has one or more user-agents and zero or more rulelines"""
+    def __init__(self):
+        self.useragents = []
+        self.rulelines = []
+        self.delay = None
+        self.req_rate = None
+
+    def __str__(self):
+        ret = []
+        for agent in self.useragents:
+            ret.append(f"User-agent: {agent}")
+        if self.delay is not None:
+            ret.append(f"Crawl-delay: {self.delay}")
+        if self.req_rate is not None:
+            rate = self.req_rate
+            ret.append(f"Request-rate: {rate.requests}/{rate.seconds}")
+        ret.extend(map(str, self.rulelines))
+        return '\n'.join(ret)
+
+    def applies_to(self, useragent):
+        """check if this entry applies to the specified agent"""
+        # split the name token and make it lower case
+        useragent = useragent.split("/")[0].lower()
+        for agent in self.useragents:
+            if agent == '*':
+                # we have the catch-all agent
+                return True
+            agent = agent.lower()
+            if agent in useragent:
+                return True
+        return False
+
+    def allowance(self, filename):
+        """Preconditions:
+        - our agent applies to this entry
+        - filename is URL decoded"""
+        for line in self.rulelines:
+            if line.applies_to(filename):
+                return line.allowance
+        return True